You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2017/05/01 20:49:42 UTC
[trafficserver] 01/02: Add new testing infra to ATS
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 7.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 57d3e8db87c11626506ccf5feafba990073ae18b
Author: Jason Kenny <dr...@live.com>
AuthorDate: Sat Apr 29 17:59:45 2017 -0500
Add new testing infra to ATS
Added new testing code
Added Apache license header
Add some getting started documentation
Doc for microserver in AuTest
update document
add execute permission to boostrap.py script
address bootstrap issue with rhel and centos
better fix for RHEL and CentOS systems
change -x to -r as 'enable' is not executable
more clean up to bootstrap.py
tweak typo
tweak test to be more stable
Update getting_started.md
Fix path to / to ./
Fix custom log test
Added 10 second delay to get AST to write out the log file
Added a verbose message to help clarify a conf file was written as expected
tweak gold files to not look for proxy-connection
Curl adds Proxy-connection on it own in some cases, and or duplicates it more than once
fix issue with virtualenv on Fedora 25
Force use of python3 for microserver
Add option to choose interface
python format clean up
create directory for TS_ROOT to allow symlinks to work. This saves space on disk
Fix the default storage config for the tests so the storage config
file is openable by nobody. Otherwise, the cachedb is not accessible
by the traffic_server user.
Big H2 file download test for issue 1646.
Add port wait on micro server startup
Fix http2 big client test
Traffic-replay client
fix testing issue with traffic_cop
Add test to see if we are shutting down
If we are, don't print message about EOL
Sync missing changes
remove via test as they are not in 7.1
---
NOTICE | 6 +
mgmt/ProcessManager.cc | 4 +-
tests/autest.sh | 35 ++
tests/bootstrap.py | 232 +++++++
tests/getting_started.md | 197 ++++++
tests/gold_tests/autest-site/copy_config.test.ext | 49 ++
tests/gold_tests/autest-site/init.cli.ext | 27 +
tests/gold_tests/autest-site/microserver.test.ext | 121 ++++
tests/gold_tests/autest-site/min_cfg/cache.config | 1 +
.../gold_tests/autest-site/min_cfg/hosting.config | 1 +
.../gold_tests/autest-site/min_cfg/ip_allow.config | 4 +
tests/gold_tests/autest-site/min_cfg/parent.config | 1 +
tests/gold_tests/autest-site/min_cfg/readme.txt | 3 +
.../gold_tests/autest-site/min_cfg/records.config | 1 +
tests/gold_tests/autest-site/min_cfg/remap.config | 1 +
.../autest-site/min_cfg/ssl_multicert.config | 1 +
.../gold_tests/autest-site/min_cfg/storage.config | 4 +
tests/gold_tests/autest-site/min_cfg/volume.config | 1 +
tests/gold_tests/autest-site/ports.py | 107 ++++
tests/gold_tests/autest-site/setup.cli.ext | 49 ++
.../gold_tests/autest-site/trafficserver.test.ext | 354 +++++++++++
tests/gold_tests/basic/basic-cop.test.py | 33 +
tests/gold_tests/basic/basic-manager.test.py | 33 +
tests/gold_tests/basic/basic.test.py | 33 +
tests/gold_tests/basic/config.test.py | 33 +
tests/gold_tests/basic/config/records_8090.config | 2 +
tests/gold_tests/basic/config/records_8091.config | 2 +
tests/gold_tests/basic/config/remap.config | 1 +
tests/gold_tests/basic/copy_config.test.py | 49 ++
tests/gold_tests/basic/copy_config2.test.py | 46 ++
tests/gold_tests/body_factory/config/remap.config | 1 +
.../body_factory/custom_response.test.py | 64 ++
.../cache/cache-generation-clear.test.py | 90 +++
.../cache/cache-generation-disjoint.test.py | 88 +++
.../cache/disjoint-wait-for-cache.test.py | 88 +++
tests/gold_tests/cache/gold/hit_default-1.gold | 8 +
tests/gold_tests/cache/gold/hit_default77.gold | 8 +
tests/gold_tests/cache/gold/hit_gen1.gold | 8 +
tests/gold_tests/cache/gold/hit_gen2.gold | 8 +
tests/gold_tests/cache/gold/miss_default-1.gold | 8 +
tests/gold_tests/cache/gold/miss_default77.gold | 8 +
tests/gold_tests/cache/gold/miss_gen1.gold | 8 +
tests/gold_tests/cache/gold/miss_gen2.gold | 8 +
tests/gold_tests/h2/gold/bigfile.gold | 12 +
tests/gold_tests/h2/gold/remap-200.gold | 4 +
tests/gold_tests/h2/h2bigclient.py | 81 +++
tests/gold_tests/h2/h2client.py | 63 ++
tests/gold_tests/h2/http2.test.py | 84 +++
tests/gold_tests/h2/ssl/server.key | 15 +
tests/gold_tests/h2/ssl/server.pem | 32 +
tests/gold_tests/logging/custom-log.test.py | 103 ++++
tests/gold_tests/logging/gold/custom.gold | 8 +
.../header_rewrite/gold/header_rewrite-303.gold | 14 +
.../header_rewrite/gold/header_rewrite-tag.gold | 1 +
.../header_rewrite/header_rewrite.test.py | 68 +++
.../pluginTest/header_rewrite/rules/rule.conf | 19 +
tests/gold_tests/remap/gold/remap-200.gold | 14 +
tests/gold_tests/remap/gold/remap-404.gold | 12 +
tests/gold_tests/remap/gold/remap-hitATS-404.gold | 11 +
tests/gold_tests/remap/gold/remap-https-200.gold | 13 +
tests/gold_tests/remap/remap_http.test.py | 91 +++
tests/gold_tests/remap/remap_https.test.py | 112 ++++
tests/gold_tests/remap/ssl/server.key | 15 +
tests/gold_tests/remap/ssl/server.pem | 32 +
tests/tools/README.md | 42 ++
tests/tools/lib/result.py | 91 +++
tests/tools/microServer/ssl/server.crt | 17 +
tests/tools/microServer/ssl/server.pem | 32 +
tests/tools/microServer/uWServer.py | 676 +++++++++++++++++++++
tests/tools/sessionvalidation/__init__.py | 17 +
tests/tools/sessionvalidation/badsession.py | 33 +
tests/tools/sessionvalidation/request.py | 47 ++
tests/tools/sessionvalidation/response.py | 38 ++
tests/tools/sessionvalidation/session.py | 44 ++
tests/tools/sessionvalidation/sessionvalidation.py | 269 ++++++++
tests/tools/sessionvalidation/transaction.py | 39 ++
tests/tools/traffic-replay/Config.py | 31 +
tests/tools/traffic-replay/RandomReplay.py | 167 +++++
tests/tools/traffic-replay/SSLReplay.py | 213 +++++++
tests/tools/traffic-replay/Scheduler.py | 62 ++
tests/tools/traffic-replay/WorkerTask.py | 45 ++
tests/tools/traffic-replay/__main__.py | 35 ++
tests/tools/traffic-replay/extractHeader.py | 64 ++
tests/tools/traffic-replay/h2Replay.py | 327 ++++++++++
tests/tools/traffic-replay/mainProcess.py | 63 ++
85 files changed, 4981 insertions(+), 1 deletion(-)
diff --git a/NOTICE b/NOTICE
index 5feedac..7209189 100644
--- a/NOTICE
+++ b/NOTICE
@@ -103,3 +103,9 @@ Copyright 2014 Google Inc. All Rights Reserved.
plugins/experimental/memcache/protocol_binary.h developed by Sun Microsystems, Inc.
Copyright (c) <2008>, Sun Microsystems, Inc. All rights reserved.
+
+~~
+
+Reusable Gold Testing System
+https://bitbucket.org/dragon512/reusable-gold-testing-system
+Copyright (c) 2015-2016 Jason Kenny All Rights Reserved.
diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc
index 78870a1..cd7c714 100644
--- a/mgmt/ProcessManager.cc
+++ b/mgmt/ProcessManager.cc
@@ -260,7 +260,9 @@ ProcessManager::pollLMConnection()
// handle EOF
if (res == 0) {
close_socket(local_manager_sockfd);
- mgmt_fatal(0, "[ProcessManager::pollLMConnection] Lost Manager EOF!");
+ if (!shutdown_event_system) {
+ mgmt_fatal(0, "[ProcessManager::pollLMConnection] Lost Manager EOF!");
+ }
}
} else if (num < 0) { /* Error */
mgmt_log("[ProcessManager::pollLMConnection] select failed or was interrupted (%d)\n", errno);
diff --git a/tests/autest.sh b/tests/autest.sh
new file mode 100755
index 0000000..45f4d51
--- /dev/null
+++ b/tests/autest.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pushd $(dirname $0) > /dev/null
+export PYTHONPATH=$(pwd):$PYTHONPATH
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+NC='\033[0m' # No Color
+if [ ! -f ./env-test/bin/autest ]; then\
+ echo -e "${RED}AuTest is not installed! Bootstrapping system...${NC}";\
+ ./bootstrap.py;\
+ echo -e "${GREEN}Done!${NC}";\
+ fi
+# this is for rhel or centos systems
+test -r /opt/rh/rh-python35/enable && . /opt/rh/rh-python35/enable
+. env-test/bin/activate
+./env-test/bin/autest -D gold_tests $*
+ret=$?
+popd > /dev/null
+exit $ret
diff --git a/tests/bootstrap.py b/tests/bootstrap.py
new file mode 100755
index 0000000..285be0a
--- /dev/null
+++ b/tests/bootstrap.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env python
+# this script sets up the testing packages to allow the tests
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import os
+import subprocess
+import platform
+import sys
+
+pip_packages = [
+ "autest",
+ "hyper"
+]
+
+
+distro_packages = {
+ "RHEL": [
+ "install epel-release",
+ "install python35",
+ "install rh-python35-python-virtualenv"
+ ],
+ "Fedora": [
+ "install python3",
+ "install python3-virtualenv",
+ "install python-virtualenv",
+ ],
+ "Ubuntu": [
+ "install python3",
+ "install python3-virtualenv",
+ "install virtualenv"
+ ],
+ "CentOS": [
+ "install epel-release",
+ "install rh-python35-python-virtualenv"
+ ]
+}
+
+
+def command_output(cmd_str):
+ print(cmd_str)
+ proc = subprocess.Popen(
+ cmd_str,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True)
+
+ # while command runs get output
+ while proc.poll() == None:
+ tmp = proc.stdout.readline()
+ sys.stdout.write(tmp)
+
+ for last_output in proc.stdout.readlines():
+ sys.stdout.write(last_output)
+
+ return proc.returncode
+
+
+def get_distro():
+ return platform.linux_distribution()
+
+
+def distro_version():
+ return int(get_distro()[1].split(".")[0])
+
+
+def isFedora():
+ return get_distro()[0].startswith("Fedora")
+
+
+def isCentOS():
+ return get_distro()[0].startswith("CentOS")
+
+
+def distro():
+ if isFedora():
+ return "Fedora"
+ if isCentOS():
+ return "CentOS"
+ if get_distro()[0].startswith("Red Hat"):
+ return "RHEL"
+ if get_distro()[0].startswith("Ubuntu"):
+ return "Ubuntu"
+
+
+def isRedHatBased():
+ return get_distro()[0].startswith("Red Hat") or get_distro()[0].startswith(
+ "Fedora") or get_distro()[0].startswith("CentOS")
+
+
+def isInstalled(prog):
+ out = subprocess.Popen(
+ ["which", prog], stdout=subprocess.PIPE).communicate()
+ if out[0] != '':
+ return True
+ return False
+
+
+def installManagerName():
+ if isRedHatBased() and distro_version() >= 22:
+ ret = "sudo dnf -y" # Fedora 22 or newer
+ elif isRedHatBased():
+ ret = "sudo yum -y" # Red Hat distro
+ else:
+ ret = "sudo apt-get -y" # Ubuntu/Debian
+
+ return ret
+
+
+def installToolName():
+ if isRedHatBased():
+ ret = "rpm -ihv" # Red Hat Based
+ else:
+ ret = "dpkg -iv" # Ubuntu/Debian
+
+ return ret
+
+
+def run_cmds(cmds):
+ for cmd in cmds:
+ # print (cmd.split[" "])
+ # subprocess.call(cmd.split[" "])
+ if command_output(cmd):
+ print("'{0}'' - Failed".format(cmd))
+
+
+def gen_package_cmds(packages):
+
+ # main install tool/manager (yum, dnf, apt-get, etc)
+ mtool = installManagerName()
+ # core install tool (rpm, dpkg, etc)
+ itool = installToolName()
+ ret = []
+
+ for p in packages:
+ if p.startswith("wget"):
+ pth = p[5:]
+ pack = os.path.split(pth)[1]
+ cmd = ["wget {0}".format(pth), "{0} ./{1}".format(itool, pack)]
+ else:
+ cmd = ["{0} {1}".format(mtool, p)]
+ ret.extend(cmd)
+ return ret
+
+
+extra=''
+if distro() == 'RHEL' or distro() == 'CentOS':
+ extra = ". /opt/rh/rh-python35/enable ;"
+
+def venv_cmds(path):
+ '''
+ Create virtual environment and add it
+ to the path being used for the script
+ '''
+
+ return [
+ # first command only needed for rhel and centos systems at this time
+ extra + " virtualenv --python=python3 {0}".format(path),
+ extra +" {0}/bin/pip install pip --upgrade".format(path)
+ ]
+
+
+
+def main():
+ " main script logic"
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--use-pip", nargs='?', default="pip", help="Which pip to use")
+
+ parser.add_argument(
+ "venv_path",
+ nargs='?',
+ default="env-test",
+ help="The directory to us to for the virtualenv")
+
+ parser.add_argument(
+ "--disable-virtualenv",
+ default=False,
+ action='store_true',
+ help="Do not create virtual environment to install packages under")
+
+ parser.add_argument(
+ '-V', '--version', action='version', version='%(prog)s 1.0.0')
+
+ args = parser.parse_args()
+ # print(args)
+ # print(get_distro())
+
+ # do we know of packages to install for the given platform
+ dist = distro()
+ cmds = []
+ if dist:
+ cmds = gen_package_cmds(distro_packages[dist])
+
+ # test to see if we should use a certain version of pip
+ path_to_pip = None
+ if args.use_pip != "pip":
+ path_to_pip = args.use_pip
+
+ # install on the system, or use virtualenv for pip based stuff
+ if not args.disable_virtualenv:
+ # Create virtual env
+ cmds += venv_cmds(args.venv_path)
+ if path_to_pip is None:
+ path_to_pip = os.path.join(args.venv_path, "bin", args.use_pip)
+
+ cmds += [extra + "{0} install {1}".format(path_to_pip, " ".join(pip_packages))]
+
+ run_cmds(cmds)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/getting_started.md b/tests/getting_started.md
new file mode 100644
index 0000000..3d36a9d
--- /dev/null
+++ b/tests/getting_started.md
@@ -0,0 +1,197 @@
+
+# Getting Started
+
+This directory contains different tests for Apache Trafficserver. It is recommended that all test move to this common area under the correct location based on the type of test being added.
+
+## Layout
+The current layout is:
+
+**gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest)
+**tools/** - contain programs used to help with testing.
+
+In the future a directory called **"unit/"** will be added for adding unit tests based on some standardized testing system.
+
+
+## Scripts
+
+To help with easy running of the tests, there is a autest.sh and bootstrap.py file.
+
+### autest.sh
+This file is a simple wrapper that will call the AuTest program in a python virtualenv. If the virtualenv is not setup it will try to install system. That will set up the Reusable Gold Testing System on most systems in a Python virtual environment. The wrapper add some basic options to the command to point to the location of the tests. Add --help for more details on options for running autest test system.
+
+### bootstrap.py
+This script should try to install python35 or better on the system, and needed python packages for running the tests.
+
+# Advance setup
+
+AuTest can be install manually instead of using the wrapper script. The advange of this is that it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done.
+1. run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script
+2. The other way is to make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ):
+ - hyper
+ - git+https://bitbucket.org/dragon512/reusable-gold-testing-system.git
+
+# Writting tests for AuTest
+When writting for the AuTest system please refer to the current documenation on the [online wiki](https://bitbucket.org/dragon512/reusable-gold-testing-system/wiki/Home) for general use of the system.
+
+## Documenation of AuTest extension for ATS.
+Autest allows for the creation of extension to help specilaize and simplify test writting for a given application domian. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added:
+
+--ats-bin < path to bin directory >
+
+This command line argument will point to your build of ATS you want to test. At this time v6.0 or newer of Trafficserver should work.
+
+### MakeATSProcess(name,command=[traffic_server],select_ports=[True])
+ * name - A name for this instance of ATS
+ * command - optional argument defining what process to use. Defaults to traffic_server.
+ * select_ports - have the testing system auto select the ports to use for this instance of ATS
+
+This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables define for making it easier to define a test.
+
+#### Environment
+The environment of the process will have a number of added environment variables to control trafficserver running the in the sandbox location correctly. This can be used to easily setup other commands that should run under same environment.
+
+##### Example
+
+```python
+# Define default ATS
+ts=Test.MakeATSProcess("ts")
+# Call traffic_ctrl to set new generation
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='traffic_ctl'
+tr.Processes.Default.ReturnCode=0
+# set the environment for traffic_control to run in to be the same as the "ts" ATS instance
+tr.Processes.Default.Env=ts.Env
+```
+
+#### Variables
+These are the current variable that are define dynamically
+
+port - the ipv4 port to listen on
+portv6 - the ipv4 port to listen on
+manager_port - the manager port used. This is set even is select_port is False
+admin_port - the admin port used. This is set even is select_port is False
+
+#### File objects
+A number of file object are define to help with adding values to a given configuration value to for a test, or testing a value exists in a log file. File that are defined currently are:
+
+##### log files
+ * squid.log
+ * error.log
+ * diags.log
+
+##### config files
+ * records.config
+ * cache.config
+ * congestion.config
+ * hosting.config
+ * icp.config
+ * ip_allow.config
+ * log_hosts.config
+ * logging.config
+ * metrics.config
+ * parent.config
+ * plugin.config
+ * remap.config
+ * socks.config
+ * splitdns.config
+ * ssl_multicert.config
+ * storage.config
+ * vaddrs.config
+ * volume.config
+
+#### Examples
+
+Create a server
+
+```python
+# don't set ports because a config file will set them
+ts1 = Test.MakeATSProcess("ts1",select_ports=False)
+ts1.Setup.ts.CopyConfig('config/records_8090.config','records.config')
+```
+
+Create a server and get the dynamic port value
+
+```python
+# Define default ATS
+ts=Test.MakeATSProcess("ts")
+#first test is a miss for default
+tr=Test.AddTestRun()
+# get port for command from Variables
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}" --verbose'.format(ts.Variables.port)
+
+```
+
+Add value to a configuration file
+```python
+# setup some config file for this server
+ts.Disk.records_config.update({
+ 'proxy.config.body_factory.enable_customizations': 3, # enable domain specific body factory
+ 'proxy.config.http.cache.generation':-1, # Start with cache turned off
+ 'proxy.config.config_update_interval_ms':1,
+ })
+ts.Disk.plugin_config.AddLine('xdebug.so')
+ts.Disk.remap_config.AddLines([
+ 'map /default/ http://127.0.0.1/ @plugin=generator.so',
+ #line 2
+ 'map /generation1/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=1' +
+ ' @plugin=generator.so',
+ #line 3
+ 'map /generation2/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=2' +
+ ' @plugin=generator.so'
+ ])
+```
+
+### CopyConfig(file, targetname=None, process=None)
+* file - name of the file to copy. Relative paths are relative from the test file location
+* targetname - the name name of the file when copied to the correct configuration location
+* process - optional process object to use for getting path location to copy to. Only needed if the Setup object call is not in the scope of the process object created with the MakeATSProcess(...) API.
+
+This function copies a given configuration file the location of a given trafficserver sandbox used in a test. Given a test might have more than on trafficserver instance, it can be difficult to understand the correct location to copy to. This function will deal with the details correctly.
+
+#### Examples
+
+Copy a file over
+
+```python
+ts1 = Test.MakeATSProcess("ts1",select_ports=False)
+# uses the setup object in the scope of the process object
+ts1.Setup.ts.CopyConfig('config/records_8090.config','records.config')
+```
+```python
+ts1 = Test.MakeATSProcess("ts1",select_ports=False)
+# uses the Setup in the global process via a variable passing
+Test.Setup.ts.CopyConfig('config/records_8090.config','records.config',ts1)
+# same as above, but uses the dynamic object model form
+Test.Setup.ts.CopyConfig('config/records_8090.config','records.config',Test.Processes.ts1)
+```
+
+## Setup Origin Server
+### Test.MakeOriginServer(Name)
+ * name - A name for this instance of Origin Server.
+
+ This function returns a AuTest process object that launches the python-based microserver. Micro-Server is a mock server which responds to client http requests. Microserver needs to be setup for the tests that require an origin server behind ATS. The server reads a JSON-formatted data file that contains request headers and the corresponding response headers. Microserver responds with payload if the response header contains Content-Length or Transfer-Enconding specified.
+
+### addResponse(filename, request_header, response_header)
+* filename - name of the file where the request header and response header will be written to in JSON format
+* request_header - dictionary of request header
+* response_header - dictionary of response header corresponding to the request header.
+
+This function adds the request header and response header to a file which is then read by the microserver to populate request-response map. The key-fields required for the header dictionary are 'headers', 'timestamp' and 'body'.
+
+### Example
+```python
+#create the origin server process
+server=Test.MakeOriginServer("server")
+#define the request header and the desired response header
+request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+#desired response form the origin server
+response_header={"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+#addResponse adds the transaction to a file which is used by the server
+server.addResponse("sessionlog.json", request_header, response_header)
+#add remap rule to traffic server
+ts.Disk.remap_config.AddLine(
+ 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+```
diff --git a/tests/gold_tests/autest-site/copy_config.test.ext b/tests/gold_tests/autest-site/copy_config.test.ext
new file mode 100644
index 0000000..2053989
--- /dev/null
+++ b/tests/gold_tests/autest-site/copy_config.test.ext
@@ -0,0 +1,49 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from future.utils import native_str
+
+class CopyATSConfig(SetupItem):
+ def __init__(self, file, targetname=None, process=None):
+ super(CopyATSConfig, self).__init__(
+ itemname="CopyATSConfig"
+ )
+ self.file=file
+ # some protection
+ if process is None and not isinstance(targetname,native_str):
+ self.process=targetname
+ else:
+ self.process=process
+ self.targetname=targetname
+
+ def setup(self):
+
+ process = self.process if self.process else self
+ try:
+ ts_dir=process.Env['TS_ROOT']
+ except:
+ if self.process:
+ raise SetupError('TS_ROOT is not defined. Cannot copy ats config file without location to copy to.')
+ else:
+ raise SetupError('TS_ROOT is not defined. Cannot copy ats config file without location to copy to. Please pass in an ATS process object')
+ config_dir = os.path.join(ts_dir,process.ComposeVariables().SYSCONFDIR.replace(process.ComposeVariables().PREFIX+"/",""))
+ host.WriteVerbose("CopyATSConfig", "Copying {0} to {1}".format(self.file, config_dir))
+ self.CopyAs(self.file,config_dir,self.targetname)
+
+AddSetupItem(CopyATSConfig, "CopyConfig", ns="ts")
diff --git a/tests/gold_tests/autest-site/init.cli.ext b/tests/gold_tests/autest-site/init.cli.ext
new file mode 100644
index 0000000..cc3a1a6
--- /dev/null
+++ b/tests/gold_tests/autest-site/init.cli.ext
@@ -0,0 +1,27 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import sys
+if sys.version_info<(3,5,0):
+ host.WriteError("You need python 3.5 or later to run these tests\n",show_stack=False)
+
+Settings.path_argument(["--ats-bin"],
+ required=True,
+ help="A user provided directory to ATS bin")
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/microserver.test.ext b/tests/gold_tests/autest-site/microserver.test.ext
new file mode 100644
index 0000000..4c39843
--- /dev/null
+++ b/tests/gold_tests/autest-site/microserver.test.ext
@@ -0,0 +1,121 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ports import get_port
+import json
+
+def addMethod(self,testName, request_header, functionName):
+ return
+
+# creates the full request or response block using headers and message data
+def httpObject(self,header,data):
+ r=dict()
+ r["timestamp"]=""
+ r["headers"]=header
+ r["body"]=data
+ return r
+
+# addResponse adds customized response with respect to request_header. request_header and response_header are both dictionaries
+def addResponse(self,filename, testName, request_header, response_header):
+
+ txn = dict()
+ txn["timestamp"] = ""
+ txn["uuid"] = testName
+ txn["request"] = request_header
+ txn["response"] = response_header
+ print("data dir",self.Variables.DataDir)
+ addTransactionToSession(txn,filename)
+ absFilepath=os.path.abspath(filename)
+ self.Setup.CopyAs(absFilepath,self.Variables.DataDir)
+ return
+
+
+
+# addResponse adds customized response with respect to request_header. request_header and response_header are both dictionaries
+def addResponse(self,filename, request_header, response_header):
+ requestline = request_header["headers"].split("\r\n")[0]
+ requestline = requestline.split(" ")[1]
+ resourceLocation = requestline.split("/",1)
+ if len(resourceLocation)>1:
+ rl = resourceLocation[1]
+ else:
+ rl = ""
+ txn = dict()
+ txn["timestamp"] = ""
+ txn["uuid"] = rl
+ txn["request"] = request_header
+ txn["response"] = response_header
+ absFilepath = os.path.join(self.Variables.DataDir, filename)
+ addTransactionToSession(txn,absFilepath)
+ #absFilepath=os.path.abspath(filename)
+ #self.Setup.CopyAs(absFilepath,self.Variables.DataDir)
+ return
+
+#adds transaction in json format to the specified file
+def addTransactionToSession(txn,JFile):
+ jsondata=None
+ if not os.path.exists(os.path.dirname(JFile)):
+ os.makedirs(os.path.dirname(JFile))
+ if os.path.exists(JFile):
+ jf = open(JFile,'r')
+ jsondata = json.load(jf)
+
+ if jsondata == None:
+ jsondata = dict()
+ jsondata["version"]='0.1'
+ jsondata["timestamp"]="1234567890.098"
+ jsondata["encoding"]="url_encoded"
+ jsondata["txns"]=list()
+ jsondata["txns"].append(txn)
+ else:
+ jsondata["txns"].append(txn)
+ with open(JFile,'w+') as jf:
+ jf.write(json.dumps(jsondata))
+
+
+#make headers with the key and values provided
+def makeHeader(self,requestString, **kwargs):
+ headerStr = requestString+'\r\n'
+ for k,v in kwargs.iteritems():
+ headerStr += k+': '+v+'\r\n'
+ headerStr = headerStr+'\r\n'
+ return headerStr
+
+
+def MakeOriginServer(obj, name,public_ip=False,options={}):
+ server_path= os.path.join(obj.Variables.AtsTestToolsDir,'microServer/uWServer.py')
+ data_dir = os.path.join(obj.RunDirectory, name)
+ # create Process
+ p = obj.Processes.Process(name)
+ port=get_port(p,"Port")
+ command = "python3 {0} --data-dir {1} --port {2} --public {3} -m test".format(server_path, data_dir, port, public_ip)
+ for flag,value in options.items() :
+ command += " {} {}".format(flag,value)
+
+ # create process
+ p.Command = command
+ p.Setup.MakeDir(data_dir)
+ p.Variables.DataDir = data_dir
+ p.Ready = When.PortOpen(port)
+ AddMethodToInstance(p,addResponse)
+ AddMethodToInstance(p,addTransactionToSession)
+
+ return p
+
+AddTestRunSet(MakeOriginServer,name="MakeOriginServer")
+AddTestRunSet(MakeOriginServer,name="MakeOrigin")
diff --git a/tests/gold_tests/autest-site/min_cfg/cache.config b/tests/gold_tests/autest-site/min_cfg/cache.config
new file mode 100644
index 0000000..6ac656b
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/cache.config
@@ -0,0 +1 @@
+# this files just needs to exist
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/min_cfg/hosting.config b/tests/gold_tests/autest-site/min_cfg/hosting.config
new file mode 100644
index 0000000..6ac656b
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/hosting.config
@@ -0,0 +1 @@
+# this files just needs to exist
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/min_cfg/ip_allow.config b/tests/gold_tests/autest-site/min_cfg/ip_allow.config
new file mode 100644
index 0000000..061bbe5
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/ip_allow.config
@@ -0,0 +1,4 @@
+src_ip=127.0.0.1 action=ip_allow method=ALL
+src_ip=::1 action=ip_allow method=ALL
+src_ip=0.0.0.0-255.255.255.255 action=ip_deny method=PUSH|PURGE|DELETE
+src_ip=::-ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff action=ip_deny method=PUSH|PURGE|DELETE
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/min_cfg/parent.config b/tests/gold_tests/autest-site/min_cfg/parent.config
new file mode 100644
index 0000000..6ac656b
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/parent.config
@@ -0,0 +1 @@
+# this files just needs to exist
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/min_cfg/readme.txt b/tests/gold_tests/autest-site/min_cfg/readme.txt
new file mode 100644
index 0000000..15a2cf8
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/readme.txt
@@ -0,0 +1,3 @@
+Contains the miniumn set of config file and setting to allow trafficserver to start in a usable way.
+The goal is to remove the need for any of these files to exist.
+Each of these files should provide an understanding of what we need to fix in the code
diff --git a/tests/gold_tests/autest-site/min_cfg/records.config b/tests/gold_tests/autest-site/min_cfg/records.config
new file mode 100644
index 0000000..8d97ae7
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/records.config
@@ -0,0 +1 @@
+# some stuff
diff --git a/tests/gold_tests/autest-site/min_cfg/remap.config b/tests/gold_tests/autest-site/min_cfg/remap.config
new file mode 100644
index 0000000..6ac656b
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/remap.config
@@ -0,0 +1 @@
+# this files just needs to exist
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/min_cfg/ssl_multicert.config b/tests/gold_tests/autest-site/min_cfg/ssl_multicert.config
new file mode 100644
index 0000000..6ac656b
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/ssl_multicert.config
@@ -0,0 +1 @@
+# this files just needs to exist
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/min_cfg/storage.config b/tests/gold_tests/autest-site/min_cfg/storage.config
new file mode 100644
index 0000000..4ebfe89
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/storage.config
@@ -0,0 +1,4 @@
+# seems good enought for doing something for playing with.
+# not good for production
+# File must exist and must have this value in it
+storage 256MB
diff --git a/tests/gold_tests/autest-site/min_cfg/volume.config b/tests/gold_tests/autest-site/min_cfg/volume.config
new file mode 100644
index 0000000..6ac656b
--- /dev/null
+++ b/tests/gold_tests/autest-site/min_cfg/volume.config
@@ -0,0 +1 @@
+# this files just needs to exist
\ No newline at end of file
diff --git a/tests/gold_tests/autest-site/ports.py b/tests/gold_tests/autest-site/ports.py
new file mode 100644
index 0000000..ce0b20e
--- /dev/null
+++ b/tests/gold_tests/autest-site/ports.py
@@ -0,0 +1,107 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+import subprocess
+
+try:
+ import queue as Queue
+except ImportError:
+ import Queue
+
+g_ports = None # ports we can use
+
+
+def PortOpen(port, address=None):
+
+ ret = False
+ if address is None:
+ address = "localhost"
+
+ address = (address, port)
+
+ try:
+ s = socket.create_connection(address, timeout=.5)
+ s.close()
+ ret = True
+ except socket.error:
+ s = None
+ ret = False
+ except socket.timeout:
+ s = None
+
+ return ret
+
+
+def setup_port_queue(amount=1000):
+ global g_ports
+ if g_ports is None:
+ g_ports = Queue.LifoQueue()
+ else:
+ return
+ try:
+ dmin, dmax = subprocess.check_output(
+ ["sysctl", "net.ipv4.ip_local_port_range"]).decode().split("=")[1].split()
+ dmin = int(dmin)
+ dmax = int(dmax)
+ except:
+ return
+
+ rmin = dmin - 2000
+ rmax = 65536 - dmax
+
+ if rmax > amount:
+ # fill in ports
+ port = dmax + 1
+ while port < 65536 and g_ports.qsize() < amount:
+ # if port good:
+ if not PortOpen(port):
+ g_ports.put(port)
+ port += 1
+ if rmin > amount and g_ports.qsize() < amount:
+ port = 2001
+ while port < dmin and g_ports.qsize() < amount:
+ # if port good:
+ if not PortOpen(port):
+ g_ports.put(port)
+ port += 1
+
+def get_port(obj, name):
+ '''
+ Get a port and set it to a variable on the object
+
+ '''
+
+ setup_port_queue()
+ if g_ports.qsize():
+ # get port
+ port = g_ports.get()
+ # assign to variable
+ obj.Variables[name] = port
+ # setup clean up step to recycle the port
+ obj.Setup.Lambda(func_cleanup=lambda: g_ports.put(
+ port), description="recycling port")
+ return port
+
+ # use old code
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.bind(('', 0)) # bind to all interfaces on an ephemeral port
+ port = sock.getsockname()[1]
+ obj.Variables[name] = port
+ return port
diff --git a/tests/gold_tests/autest-site/setup.cli.ext b/tests/gold_tests/autest-site/setup.cli.ext
new file mode 100644
index 0000000..6f38a87
--- /dev/null
+++ b/tests/gold_tests/autest-site/setup.cli.ext
@@ -0,0 +1,49 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json, subprocess
+import pprint
+
+if Arguments.ats_bin is not None:
+ # Add environment variables
+ ENV['ATS_BIN'] = Arguments.ats_bin
+ #if Arguments.ats_bin not in ENV['PATH']:
+ #ENV['PATH'] = Arguments.ats_bin + ':' + ENV['PATH']
+
+if ENV['ATS_BIN'] is not None:
+ # Add variables for Tests
+ traffic_layout = os.path.join(ENV['ATS_BIN'], "traffic_layout")
+ if not os.path.isdir(ENV['ATS_BIN']):
+ host.WriteError("--ats-bin requires a directory", show_stack=False)
+ if not os.path.isfile(traffic_layout):
+ host.WriteError("traffic_layout is not found. Aborting tests - Bad build or install.", show_stack=False)
+ try:
+ out = subprocess.check_output([traffic_layout, "--json"])
+ except subprocess.CalledProcessError:
+ host.WriteError("traffic_layout is broken. Aborting tests - The build of traffic server is bad.", show_stack=False)
+ out = json.loads(out.decode("utf-8"))
+ for k,v in out.items():
+ out[k]=v[:-1] if v.endswith('/') else v
+ Variables.update(out)
+ host.WriteVerbose(['ats'],"Traffic server layout Data:\n",pprint.pformat(out))
+
+Variables.AtsTestToolsDir = os.path.join(AutestSitePath,'../../tools')
+
+# modify delay times as we always have to kill Trafficserver
+# no need to wait
+Variables.Autest.StopProcessLongDelaySeconds=0
diff --git a/tests/gold_tests/autest-site/trafficserver.test.ext b/tests/gold_tests/autest-site/trafficserver.test.ext
new file mode 100644
index 0000000..495f164
--- /dev/null
+++ b/tests/gold_tests/autest-site/trafficserver.test.ext
@@ -0,0 +1,354 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import socket
+from ports import get_port
+
+
+def make_id(s):
+ return s.replace(".", "_").replace('-', '_')
+
+# this forms is for the global process define
+
+
+def MakeATSProcess(obj, name, command='traffic_server', select_ports=True):
+ #####################################
+ # common locations
+
+ # directory we will setup for the ATS to run under
+ ts_dir = os.path.join(obj.RunDirectory, name)
+ # common bin directory
+ bin_dir = 'bin'
+ # ideally we would use a value like config.. but there is a bug
+ # in which records.conf values are not loaded correctly from this location,
+ # so we use the expecetd "build" layout to correct the issue for the time
+ # being
+ config_dir = os.path.join(
+ ts_dir,
+ obj.Variables.SYSCONFDIR.replace(
+ obj.Variables.PREFIX + "/",
+ ""
+ )
+ )
+ # directory contains the html response templates
+ template_dir = os.path.join(config_dir, "body_factory")
+ # contains plugins
+ plugin_dir = os.path.join(
+ ts_dir, obj.Variables.PLUGINDIR.replace(obj.Variables.PREFIX + "/", ""))
+ # the log directory
+ log_dir = os.path.join(ts_dir, 'log')
+ runtime_dir = os.path.join(
+ ts_dir, obj.Variables.RUNTIMEDIR.replace(obj.Variables.PREFIX + "/", ""))
+ ssl_dir = os.path.join(ts_dir, 'ssl')
+ storage_dir = os.path.join(ts_dir, 'storage')
+ # create process
+ p = obj.Processes.Process(name, command)
+
+ # we want to have a few directories more fixed
+ # this helps with debugging as location are common
+ # we do this by overiding locations from the "layout"
+ # used as part of build. This means loctaion such as
+ # PROXY_CONFIG_BIN_PATH with alway be $root/bin
+ # not something else such as bin64
+ #####
+
+ # set root for this test
+ p.Env['TS_ROOT'] = ts_dir
+ p.Setup.MakeDir(ts_dir)
+
+ # set bin location
+
+ p.Env['PROXY_CONFIG_BIN_PATH'] = bin_dir
+ bin_path = os.path.join(ts_dir, bin_dir)
+ p.Env['PATH'] = bin_path + os.pathsep + p.ComposeEnv()['PATH']
+ p.Setup.Copy(p.Variables.BINDIR, bin_path, True)
+
+ #########################################################
+ # setup config directory
+
+ # copy all basic config files we need to get this to work
+ cfg_dir = os.path.join(AUTEST_SITE_PATH, "min_cfg")
+ for f in os.listdir(cfg_dir):
+ p.Setup.CopyAs(os.path.join(cfg_dir, f), config_dir)
+
+ #########################################################
+ # setup read-only data directory in config. Needed for response body
+ # reponses
+
+ p.Env['PROXY_CONFIG_BODY_FACTORY_TEMPLATE_SETS_DIR'] = template_dir
+ p.Variables.body_factory_template_dir = template_dir
+ p.Setup.Copy(os.path.join(p.Variables.SYSCONFDIR,
+ 'body_factory'), template_dir)
+
+ #########################################################
+ # setup read-only data directory for plugins
+
+ p.Env['PROXY_CONFIG_PLUGIN_PLUGIN_DIR'] = plugin_dir
+ p.Setup.Copy(p.Variables.PLUGINDIR, plugin_dir)
+
+ #########################################################
+ # create subdirectories that need to exist (but are empty)
+ # log directory has to be created with correct permissions
+ p.Setup.MakeDir(log_dir) # log directory has to be created
+ p.Setup.Chown(log_dir, "nobody", "nobody", ignore=True)
+
+ # set env so traffic server uses correct locations
+ p.Env['PROXY_CONFIG_LOG_LOGFILE_DIR'] = log_dir
+ p.Variables.LOGDIR = log_dir
+
+ # this is needed for cache and communication sockets
+ # Below was to make shorter paths but the code in
+ # traffic_ctl is broken and ignores this.
+ # p.Env['PROXY_CONFIG_LOCAL_STATE_DIR']=runtime_dir
+ p.Env['PROXY_CONFIG_HOSTDB_STORAGE_PATH'] = runtime_dir
+ # p.Variables.RUNTIMEDIR=runtime_dir
+ p.Setup.MakeDir(runtime_dir)
+ # will need this for traffic_manager is it runs
+ p.Setup.MakeDir(os.path.join(config_dir, 'snapshots'))
+
+ ##########################################################
+ # create subdirectories that need to exist (but are empty)
+ # ssl directory has to be created for keeping certs and keys
+ p.Setup.MakeDir(ssl_dir)
+ p.Setup.Chown(ssl_dir, "nobody", "nobody", ignore=True)
+
+ # set env so traffic server uses correct locations
+ p.Env['PROXY_CONFIG_SSL_DIR'] = ssl_dir
+ p.Variables.SSLDir = ssl_dir
+ AddMethodToInstance(p, addSSLfile)
+ ########################################################
+ # cache.db directory
+ p.Setup.MakeDir(storage_dir)
+ p.Setup.Chown(storage_dir, "nobody", "nobody", ignore=True)
+
+ # set env so traffic server uses correct locations
+ p.Env['PROXY_CONFIG_STORAGE_DIR'] = storage_dir
+ p.Variables.STORAGEDIR = storage_dir
+ #########################################################
+ # define the basic file for a given test run
+ # traffic.out ?? # cannot find it at the moment...
+ # squid.log
+ fname = "squid.log"
+ tmpname = os.path.join(log_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname))
+ # error.log
+ fname = "error.log"
+ tmpname = os.path.join(log_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), exists=False)
+ # diags.log
+ fname = "diags.log"
+ tmpname = os.path.join(log_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname))
+ # add this test back once we have network namespaces working again
+ p.Disk.diags_log.Content = Testers.ExcludesExpression(
+ "ERROR:", "diags.log should not contain errors")
+ p.Disk.diags_log.Content += Testers.ExcludesExpression(
+ "FATAL:", "diags.log should not contain errors")
+
+ # config files
+ fname = "records.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config:records")
+
+ fname = "cache.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "congestion.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "hosting.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "icp.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "ip_allow.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "log_hosts.config"
+ tmpname = os.path.join(config_dir, fname)
+ # p.Disk.File(tmpname,id=make_id(fname),typename="ats:config").AddLine("#
+ # need something in here")
+
+ # magic file that should probally not exist
+ fname = "logging.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "metrics.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "parent.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "plugin.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "remap.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "socks.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "splitdns.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "ssl_multicert.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "storage.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "vaddrs.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ fname = "volume.config"
+ tmpname = os.path.join(config_dir, fname)
+ p.Disk.File(tmpname, id=make_id(fname), typename="ats:config")
+
+ ##########################################################
+ # set up default ports
+ # get some ports TODO make it so we can hold on to the socket
+ if select_ports:
+ # some system have a bug in which ipv4 and ipv6 share port space
+ # Make two different ports to avoid this
+ get_port(p, "port")
+ get_port(p, "portv6")
+ p.Ready = When.PortOpen(p.Variables.port)
+ else:
+ p.Variables.port = 8080
+ p.Variables.portv6 = 8080
+ get_port(p, "manager_port")
+ get_port(p, "admin_port")
+
+ # set the ports
+ if select_ports:
+ p.Env['PROXY_CONFIG_HTTP_SERVER_PORTS'] = "{0} {1}:ipv6".format(
+ p.Variables.port, p.Variables.portv6) # your own listen port
+ p.Env['PROXY_CONFIG_PROCESS_MANAGER_MGMT_PORT'] = str(
+ p.Variables.manager_port)
+ p.Env['PROXY_CONFIG_ADMIN_SYNTHETIC_PORT'] = str(p.Variables.admin_port)
+ p.Env['PROXY_CONFIG_ADMIN_AUTOCONF_PORT'] = str(
+ p.Variables.admin_port) # support pre ATS 6.x
+
+ # since we always kill this
+ p.ReturnCode = None
+
+ return p
+
+##################################
+# added to ats process object to help deal with config files
+
+
+class Config(File):
+ '''
+ Class to represent a config file
+ '''
+
+ def __init__(self, runable, name, exists=None, size=None, content_tester=None, execute=False, runtime=True, content=None):
+ super(Config, self).__init__(
+ runable, name, exists=None, size=None, content_tester=None, execute=False, runtime=True
+ )
+
+ self.content = content
+ self._added = False
+
+ def AddLines(self, lines):
+ for line in lines:
+ self.AddLine(line)
+
+ def _do_write(self, name):
+ '''
+ Write contents to disk
+ '''
+ host.WriteVerbosef('ats-config-file',
+ "Writting out file {0}", self.Name)
+ if self.content is not None:
+ with open(name, 'w') as f:
+ f.write(self.content)
+ return (False, "Appended file {0}".format(self.Name), "Success")
+
+ def AddLine(self, line):
+ if not self._added:
+ self.WriteCustomOn(self._do_write)
+ self._added = True
+ if self.content is None:
+ self.content = ""
+ if not line.endswith('\n'):
+ line += '\n'
+ self.content += line
+
+
+class RecordsConfig(Config, dict):
+ '''
+ Create a "dict" representation of records.config
+
+ This can be accessed as a multi-level dictionary
+
+ such as:
+ rc['CONFIG']['proxy.config.log.hostname']
+ '''
+
+ reverse_kind_map = {str: 'STRING',
+ int: 'INT',
+ float: 'FLOAT',
+ }
+
+ line_template = 'CONFIG {name} {kind} {val}\n'
+
+ def __init__(self, runable, name, exists=None, size=None, content_tester=None, execute=False, runtime=True):
+ super(RecordsConfig, self).__init__(
+ runable, name, exists=None, size=None, content_tester=None, execute=False, runtime=True
+ )
+ self.WriteCustomOn(self._do_write)
+
+ def _do_write(self, name):
+ host.WriteVerbosef('ats-config-file', "Writting out file {0}", name)
+ if len(self) > 0:
+ with open(name, 'w') as f:
+ for name, val in self.items():
+ f.write(self.line_template.format(name=name,
+ kind=self.reverse_kind_map[
+ type(val)],
+ val=val))
+ return (False, "Writing config file {0}".format(os.path.split(self.Name)[-1]), "Success")
+##########################################################################
+
+
+def addSSLfile(self, filename):
+ self.Setup.CopyAs(filename, self.Variables.SSLDir)
+
+RegisterFileType(Config, "ats:config")
+RegisterFileType(RecordsConfig, "ats:config:records")
+AddTestRunSet(MakeATSProcess, name="MakeATSProcess")
diff --git a/tests/gold_tests/basic/basic-cop.test.py b/tests/gold_tests/basic/basic-cop.test.py
new file mode 100644
index 0000000..fa87654
--- /dev/null
+++ b/tests/gold_tests/basic/basic-cop.test.py
@@ -0,0 +1,33 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = '''
+Test that Trafficserver starts with default configurations.
+'''
+
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+
+p=Test.MakeATSProcess("ts", command="traffic_cop --debug --stdout", select_ports=False)
+t = Test.AddTestRun("Test traffic server started properly")
+t.StillRunningAfter = Test.Processes.ts
+
+p = t.Processes.Default
+p.Command = "curl http://127.0.0.1:8080"
+p.ReturnCode = 0
+p.StartBefore(Test.Processes.ts, ready = When.PortOpen(8080))
+
diff --git a/tests/gold_tests/basic/basic-manager.test.py b/tests/gold_tests/basic/basic-manager.test.py
new file mode 100644
index 0000000..a9eba74
--- /dev/null
+++ b/tests/gold_tests/basic/basic-manager.test.py
@@ -0,0 +1,33 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = '''
+Test that Trafficserver starts with default configurations.
+'''
+
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+
+p=Test.MakeATSProcess("ts", command="traffic_manager",select_ports=False)
+t = Test.AddTestRun("Test traffic server started properly")
+t.StillRunningAfter = Test.Processes.ts
+
+p = t.Processes.Default
+p.Command = "curl http://127.0.0.1:8080"
+p.ReturnCode = 0
+p.StartBefore(Test.Processes.ts, ready = When.PortOpen(8080))
+
diff --git a/tests/gold_tests/basic/basic.test.py b/tests/gold_tests/basic/basic.test.py
new file mode 100644
index 0000000..7dcbcd2
--- /dev/null
+++ b/tests/gold_tests/basic/basic.test.py
@@ -0,0 +1,33 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = '''
+Test that Trafficserver starts with default configurations.
+'''
+
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+
+p=Test.MakeATSProcess("ts",select_ports=False)
+t = Test.AddTestRun("Test traffic server started properly")
+t.StillRunningAfter = Test.Processes.ts
+
+p = t.Processes.Default
+p.Command = "curl http://127.0.0.1:8080"
+p.ReturnCode = 0
+p.StartBefore(Test.Processes.ts, ready = When.PortOpen(8080))
+
diff --git a/tests/gold_tests/basic/config.test.py b/tests/gold_tests/basic/config.test.py
new file mode 100644
index 0000000..48f3c73
--- /dev/null
+++ b/tests/gold_tests/basic/config.test.py
@@ -0,0 +1,33 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = "Test start up of Traffic server with configuration modification of starting port"
+
+Test.SkipUnless(Condition.HasProgram("curl",
+ "Curl needs to be installed on your system for this test to work"))
+
+ts1 = Test.MakeATSProcess("ts1",select_ports=False)
+ts1.Setup.ts.CopyConfig('config/records_8090.config',"records.config")
+t = Test.AddTestRun("Test traffic server started properly")
+t.StillRunningAfter = ts1
+
+p = t.Processes.Default
+p.Command = "curl 127.0.0.1:8090"
+p.ReturnCode = 0
+
+p.StartBefore(Test.Processes.ts1, ready = When.PortOpen(8090))
\ No newline at end of file
diff --git a/tests/gold_tests/basic/config/records_8090.config b/tests/gold_tests/basic/config/records_8090.config
new file mode 100644
index 0000000..ca56e77
--- /dev/null
+++ b/tests/gold_tests/basic/config/records_8090.config
@@ -0,0 +1,2 @@
+CONFIG proxy.config.http.server_ports STRING 8090
+
diff --git a/tests/gold_tests/basic/config/records_8091.config b/tests/gold_tests/basic/config/records_8091.config
new file mode 100644
index 0000000..1b560d8
--- /dev/null
+++ b/tests/gold_tests/basic/config/records_8091.config
@@ -0,0 +1,2 @@
+CONFIG proxy.config.http.server_ports STRING 8091
+
diff --git a/tests/gold_tests/basic/config/remap.config b/tests/gold_tests/basic/config/remap.config
new file mode 100644
index 0000000..553b66b
--- /dev/null
+++ b/tests/gold_tests/basic/config/remap.config
@@ -0,0 +1 @@
+regex_map http://(.*)/ http://localhost:9999/
\ No newline at end of file
diff --git a/tests/gold_tests/basic/copy_config.test.py b/tests/gold_tests/basic/copy_config.test.py
new file mode 100644
index 0000000..7543be3
--- /dev/null
+++ b/tests/gold_tests/basic/copy_config.test.py
@@ -0,0 +1,49 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = "Test start up of Traffic server with configuration modification of starting port of different servers at the same time"
+
+Test.SkipUnless(Condition.HasProgram("curl",
+ "Curl needs to be installed on your system for this test to work"))
+
+# set up some ATS processes
+ts1 = Test.MakeATSProcess("ts1",select_ports=False)
+ts1.Setup.ts.CopyConfig('config/records_8090.config','records.config')
+
+ts2 = Test.MakeATSProcess("ts2",select_ports=False)
+ts2.Setup.ts.CopyConfig('config/records_8091.config','records.config')
+
+# setup a testrun
+t = Test.AddTestRun("Talk to ts1")
+t.StillRunningAfter = ts1
+t.StillRunningAfter += ts2
+p = t.Processes.Default
+p.Command = "curl 127.0.0.1:8090"
+p.ReturnCode = 0
+p.StartBefore(Test.Processes.ts1, ready = When.PortOpen(8090))
+p.StartBefore(Test.Processes.ts2, ready = When.PortOpen(8091))
+
+# setup a testrun
+t = Test.AddTestRun("Talk to ts2")
+t.StillRunningBefore = ts1
+t.StillRunningBefore += ts2
+t.StillRunningAfter = ts1
+t.StillRunningAfter += ts2
+p = t.Processes.Default
+p.Command = "curl 127.0.0.1:8091"
+p.ReturnCode = 0
\ No newline at end of file
diff --git a/tests/gold_tests/basic/copy_config2.test.py b/tests/gold_tests/basic/copy_config2.test.py
new file mode 100644
index 0000000..65c3ad0
--- /dev/null
+++ b/tests/gold_tests/basic/copy_config2.test.py
@@ -0,0 +1,46 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = "Test start up of Traffic server with generated ports of more than one servers at the same time"
+
+Test.SkipUnless(Condition.HasProgram("curl",
+ "Curl needs to be installed on your system for this test to work"))
+
+# set up some ATS processes
+ts1 = Test.MakeATSProcess("ts1")
+ts2 = Test.MakeATSProcess("ts2")
+
+# setup a testrun
+t = Test.AddTestRun("Talk to ts1")
+t.StillRunningAfter = ts1
+t.StillRunningAfter += ts2
+p = t.Processes.Default
+p.Command = "curl 127.0.0.1:{0}".format(ts1.Variables.port)
+p.ReturnCode = 0
+p.StartBefore(Test.Processes.ts1, ready = When.PortOpen(ts1.Variables.port))
+p.StartBefore(Test.Processes.ts2, ready = When.PortOpen(ts2.Variables.port))
+
+# setup a testrun
+t = Test.AddTestRun("Talk to ts2")
+t.StillRunningBefore = ts1
+t.StillRunningBefore += ts2
+t.StillRunningAfter = ts1
+t.StillRunningAfter += ts2
+p = t.Processes.Default
+p.Command = "curl 127.0.0.1:{0}".format(ts2.Variables.port)
+p.ReturnCode = 0
\ No newline at end of file
diff --git a/tests/gold_tests/body_factory/config/remap.config b/tests/gold_tests/body_factory/config/remap.config
new file mode 100644
index 0000000..553b66b
--- /dev/null
+++ b/tests/gold_tests/body_factory/config/remap.config
@@ -0,0 +1 @@
+regex_map http://(.*)/ http://localhost:9999/
\ No newline at end of file
diff --git a/tests/gold_tests/body_factory/custom_response.test.py b/tests/gold_tests/body_factory/custom_response.test.py
new file mode 100644
index 0000000..18350ac
--- /dev/null
+++ b/tests/gold_tests/body_factory/custom_response.test.py
@@ -0,0 +1,64 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+Test.Summary = '''
+Tests for custom reponse body
+'''
+
+# this test currently fails and it should not
+Test.SkipIf(Condition.true("This test fails at the moment as is turned off"))
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+
+ts=Test.MakeATSProcess("ts")
+ts.Disk.records_config.update({
+ 'proxy.config.body_factory.enable_customizations': 3, # enable domain specific body factory
+ })
+ts.Disk.remap_config.AddLine(
+ 'map / http://www.linkedin.com/ @action=deny'
+ )
+
+
+domain_directory = ['www.linkedin.com', '127.0.0.1', 'www.foobar.net']
+body_factory_dir=ts.Variables.body_factory_template_dir
+# for each domain
+set=False
+for directory_item in domain_directory:
+ # write out a files with some content for Traffic server for given domain
+ ts.Disk.File(os.path.join(body_factory_dir, directory_item, "access#denied")).\
+ WriteOn("{0} 44 Not 89 found".format(directory_item))
+
+ ts.Disk.File(os.path.join(body_factory_dir, directory_item, ".body_factory_info")).\
+ WriteOn("")
+ # make a test run for a given domain
+ tr=Test.AddTestRun("Test domain {0}".format(directory_item))
+ if not set:
+ #Start the ATS process for first test run
+ tr.Processes.Default.StartBefore(Test.Processes.ts)
+ set = True
+ tr.StillRunningAfter = ts
+ else:
+ # test that ats is still running before and after
+ tr.StillRunningBefore = ts
+ tr.StillRunningAfter = ts
+
+ tr.Processes.Default.Command="curl --proxy 127.0.0.1:{1} {0}".format(directory_item,ts.Variables.port)
+ tr.Processes.Default.ReturnCode=0
+ tr.Streams.All=Testers.ContainsExpression("{0} Not found".format(directory_item),"should contain custom data")
+
diff --git a/tests/gold_tests/cache/cache-generation-clear.test.py b/tests/gold_tests/cache/cache-generation-clear.test.py
new file mode 100644
index 0000000..f36a339
--- /dev/null
+++ b/tests/gold_tests/cache/cache-generation-clear.test.py
@@ -0,0 +1,90 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+Test.Summary = '''
+Test that incrementing the cache generation acts like a cache clear
+'''
+# need Curl
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts",command="traffic_manager")
+
+# setup some config file for this server
+ts.Disk.records_config.update({
+ 'proxy.config.body_factory.enable_customizations': 3, # enable domain specific body factory
+ 'proxy.config.http.cache.generation':-1, # Start with cache turned off
+ 'proxy.config.config_update_interval_ms':1,
+ })
+ts.Disk.plugin_config.AddLine('xdebug.so')
+ts.Disk.remap_config.AddLines([
+ 'map /default/ http://127.0.0.1/ @plugin=generator.so',
+ #line 2
+ 'map /generation1/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=1' +
+ ' @plugin=generator.so',
+ #line 3
+ 'map /generation2/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=2' +
+ ' @plugin=generator.so'
+ ])
+
+objectid = uuid.uuid4()
+#first test is a miss for default
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(Test.Processes.ts,ready=5)
+tr.Processes.Default.Streams.All="gold/miss_default-1.gold"
+
+# Second touch is a HIT for default.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_default-1.gold"
+
+# Call traffic_ctrl to set new generation
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='traffic_ctl --debug config set proxy.config.http.cache.generation 77'
+tr.Processes.Default.ForceUseShell=False
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Env=ts.Env # set the environment for traffic_control to run in
+
+# new generation should first be a miss.
+tr=Test.AddTestRun()
+tr.DelayStart=15 # delay start of test run to allow previous command to take effect
+# create a new traffic_ctrl call and the environment
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/miss_default77.gold"
+
+# new generation should should now hit.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_default77.gold"
+
+# should still hit.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_default77.gold"
+
diff --git a/tests/gold_tests/cache/cache-generation-disjoint.test.py b/tests/gold_tests/cache/cache-generation-disjoint.test.py
new file mode 100644
index 0000000..39c0b67
--- /dev/null
+++ b/tests/gold_tests/cache/cache-generation-disjoint.test.py
@@ -0,0 +1,88 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+Test.Summary = '''
+Test that the same URL path in different cache generations creates disjoint objects
+'''
+# need Curl
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts")
+
+# setup some config file for this server
+ts.Disk.records_config.update({
+ 'proxy.config.body_factory.enable_customizations': 3, # enable domain specific body factory
+ 'proxy.config.http.cache.generation':-1, # Start with cache turned off
+ 'proxy.config.http.wait_for_cache': 1,
+ 'proxy.config.config_update_interval_ms':1,
+
+ })
+ts.Disk.plugin_config.AddLine('xdebug.so')
+ts.Disk.remap_config.AddLines([
+ 'map /default/ http://127.0.0.1/ @plugin=generator.so',
+ #line 2
+ 'map /generation1/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=1' +
+ ' @plugin=generator.so',
+ #line 3
+ 'map /generation2/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=2' +
+ ' @plugin=generator.so'
+ ])
+
+objectid = uuid.uuid4()
+#first test is a miss for default
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(Test.Processes.ts,ready=2)
+tr.Processes.Default.Streams.All="gold/miss_default-1.gold"
+
+# Same URL in generation 1 is a MISS.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/miss_gen1.gold"
+
+# Same URL in generation 2 is still a MISS.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/miss_gen2.gold"
+
+# Second touch is a HIT for default.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_default-1.gold"
+
+# Second touch is a HIT for generation1.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_gen1.gold"
+
+# Second touch is a HIT for generation2.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_gen2.gold"
diff --git a/tests/gold_tests/cache/disjoint-wait-for-cache.test.py b/tests/gold_tests/cache/disjoint-wait-for-cache.test.py
new file mode 100644
index 0000000..cc45877
--- /dev/null
+++ b/tests/gold_tests/cache/disjoint-wait-for-cache.test.py
@@ -0,0 +1,88 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+Test.Summary = '''
+Same as cache-generaertaion-disjoint, but uses proxy.config.http.wait_for_cache which should delay
+the server from accepting connection till the cache is loaded
+'''
+# need Curl
+Test.SkipUnless(Condition.HasProgram("curl","Curl need to be installed on system for this test to work"))
+Test.SkipIf(Condition.true("This test fails at the moment as is turned off"))
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts")
+
+# setup some config file for this server
+ts.Disk.records_config.update({
+ 'proxy.config.body_factory.enable_customizations': 3, # enable domain specific body factory
+ 'proxy.config.http.cache.generation':-1, # Start with cache turned off
+ 'proxy.config.config_update_interval_ms':1,
+ 'proxy.config.http.wait_for_cache':3,
+ })
+ts.Disk.plugin_config.AddLine('xdebug.so')
+ts.Disk.remap_config.AddLines([
+ 'map /default/ http://127.0.0.1/ @plugin=generator.so',
+ #line 2
+ 'map /generation1/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=1' +
+ ' @plugin=generator.so',
+ #line 3
+ 'map /generation2/ http://127.0.0.1/' +
+ ' @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=2' +
+ ' @plugin=generator.so'
+ ])
+
+objectid = uuid.uuid4()
+#first test is a miss for default
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.StartBefore(Test.Processes.ts)
+tr.Processes.Default.Streams.All="gold/miss_default-1.gold"
+
+# Same URL in generation 1 is a MISS.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/miss_gen1.gold"
+
+# Same URL in generation 2 is still a MISS.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/miss_gen2.gold"
+
+# Second touch is a HIT for default.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_default-1.gold"
+
+# Second touch is a HIT for generation1.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_gen1.gold"
+
+# Second touch is a HIT for generation2.
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format(ts.Variables.port,objectid)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.All="gold/hit_gen2.gold"
diff --git a/tests/gold_tests/cache/gold/hit_default-1.gold b/tests/gold_tests/cache/gold/hit_default-1.gold
new file mode 100644
index 0000000..55590ff
--- /dev/null
+++ b/tests/gold_tests/cache/gold/hit_default-1.gold
@@ -0,0 +1,8 @@
+{}
+> GET /default/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: hit-fresh
+< X-Cache-Generation: -1
+{}
diff --git a/tests/gold_tests/cache/gold/hit_default77.gold b/tests/gold_tests/cache/gold/hit_default77.gold
new file mode 100644
index 0000000..79f4dc8
--- /dev/null
+++ b/tests/gold_tests/cache/gold/hit_default77.gold
@@ -0,0 +1,8 @@
+{}
+> GET /default/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: hit-fresh
+< X-Cache-Generation: 77
+{}
diff --git a/tests/gold_tests/cache/gold/hit_gen1.gold b/tests/gold_tests/cache/gold/hit_gen1.gold
new file mode 100644
index 0000000..0830f99
--- /dev/null
+++ b/tests/gold_tests/cache/gold/hit_gen1.gold
@@ -0,0 +1,8 @@
+{}
+> GET /generation1/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: hit-fresh
+< X-Cache-Generation: 1
+{}
diff --git a/tests/gold_tests/cache/gold/hit_gen2.gold b/tests/gold_tests/cache/gold/hit_gen2.gold
new file mode 100644
index 0000000..ee30558
--- /dev/null
+++ b/tests/gold_tests/cache/gold/hit_gen2.gold
@@ -0,0 +1,8 @@
+{}
+> GET /generation2/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: hit-fresh
+< X-Cache-Generation: 2
+{}
diff --git a/tests/gold_tests/cache/gold/miss_default-1.gold b/tests/gold_tests/cache/gold/miss_default-1.gold
new file mode 100644
index 0000000..ada9842
--- /dev/null
+++ b/tests/gold_tests/cache/gold/miss_default-1.gold
@@ -0,0 +1,8 @@
+{}
+> GET /default/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: miss
+< X-Cache-Generation: -1
+{}
diff --git a/tests/gold_tests/cache/gold/miss_default77.gold b/tests/gold_tests/cache/gold/miss_default77.gold
new file mode 100644
index 0000000..083ff08
--- /dev/null
+++ b/tests/gold_tests/cache/gold/miss_default77.gold
@@ -0,0 +1,8 @@
+{}
+> GET /default/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: miss
+< X-Cache-Generation: 77
+{}
diff --git a/tests/gold_tests/cache/gold/miss_gen1.gold b/tests/gold_tests/cache/gold/miss_gen1.gold
new file mode 100644
index 0000000..cacf5c3
--- /dev/null
+++ b/tests/gold_tests/cache/gold/miss_gen1.gold
@@ -0,0 +1,8 @@
+{}
+> GET /generation1/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: miss
+< X-Cache-Generation: 1
+{}
diff --git a/tests/gold_tests/cache/gold/miss_gen2.gold b/tests/gold_tests/cache/gold/miss_gen2.gold
new file mode 100644
index 0000000..cf1eb42
--- /dev/null
+++ b/tests/gold_tests/cache/gold/miss_gen2.gold
@@ -0,0 +1,8 @@
+{}
+> GET /generation2/cache/10/{} HTTP/1.1
+{}
+< Server: ATS/{}
+< X-Cache-Key: http://127.0.0.1/cache/10/{}
+< X-Cache: miss
+< X-Cache-Generation: 2
+{}
diff --git a/tests/gold_tests/h2/gold/bigfile.gold b/tests/gold_tests/h2/gold/bigfile.gold
new file mode 100644
index 0000000..5fd9215
--- /dev/null
+++ b/tests/gold_tests/h2/gold/bigfile.gold
@@ -0,0 +1,12 @@
+Content length = 191414
+
+Body length = 191414
+
+Content success
+
+Content length = 191414
+
+Body length = 191414
+
+Content success
+
diff --git a/tests/gold_tests/h2/gold/remap-200.gold b/tests/gold_tests/h2/gold/remap-200.gold
new file mode 100644
index 0000000..5f7e6ec
--- /dev/null
+++ b/tests/gold_tests/h2/gold/remap-200.gold
@@ -0,0 +1,4 @@
+HTTP/2 200
+date: {}
+server: ATS/{}
+
diff --git a/tests/gold_tests/h2/h2bigclient.py b/tests/gold_tests/h2/h2bigclient.py
new file mode 100644
index 0000000..245ab14
--- /dev/null
+++ b/tests/gold_tests/h2/h2bigclient.py
@@ -0,0 +1,81 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from hyper import HTTPConnection
+import hyper
+import argparse
+
+def getResponseString(response):
+ typestr = str(type(response))
+ if typestr.find('HTTP20') != -1:
+ string = "HTTP/2 {0}\r\n".format(response.status)
+ else:
+ string = "HTTP {0}\r\n".format(response.status)
+ string+='date: '+response.headers.get('date')[0].decode('utf-8')+"\r\n"
+ string+='server: '+response.headers.get('Server')[0].decode('utf-8')+"\r\n"
+ return string
+
+def makerequest(port):
+ hyper.tls._context = hyper.tls.init_context()
+ hyper.tls._context.check_hostname = False
+ hyper.tls._context.verify_mode = hyper.compat.ssl.CERT_NONE
+
+ conn = HTTPConnection('localhost:{0}'.format(port), secure=True)
+
+ # Fetch the object twice so we know at least one time comes from cache
+ # Exploring timing options
+ sites=['/bigfile', '/bigfile']
+ responses = []
+ request_ids = []
+ for site in sites:
+ request_id = conn.request('GET',url=site)
+ request_ids.append(request_id)
+
+ # get responses
+ for req_id in request_ids:
+ response = conn.get_response(req_id)
+ body = response.read()
+ cl = response.headers.get('Content-Length')[0]
+ print("Content length = {}\r\n".format(int(cl)))
+ print("Body length = {}\r\n".format(len(body)))
+ error=0;
+ if chr(body[0]) != 'a':
+ error = 1
+ print("First char {}".format(body[0]))
+ i = 1
+ while i < len(body) and not error:
+ error = chr(body[i]) != 'b'
+ if error:
+ print("bad char {} at {}".format(body[i], i))
+ i = i + 1
+ if not error:
+ print("Content success\r\n")
+ else :
+ print("Content fail\r\n")
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--port","-p",
+ type=int,
+ help="Port to use")
+ args=parser.parse_args()
+ makerequest(args.port)
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/gold_tests/h2/h2client.py b/tests/gold_tests/h2/h2client.py
new file mode 100644
index 0000000..1654f59
--- /dev/null
+++ b/tests/gold_tests/h2/h2client.py
@@ -0,0 +1,63 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from hyper import HTTPConnection
+import hyper
+import argparse
+
+def getResponseString(response):
+ typestr = str(type(response))
+ if typestr.find('HTTP20') != -1:
+ string = "HTTP/2 {0}\r\n".format(response.status)
+ else:
+ string = "HTTP {0}\r\n".format(response.status)
+ string+='date: '+response.headers.get('date')[0].decode('utf-8')+"\r\n"
+ string+='server: '+response.headers.get('Server')[0].decode('utf-8')+"\r\n"
+ return string
+
+def makerequest(port):
+ hyper.tls._context = hyper.tls.init_context()
+ hyper.tls._context.check_hostname = False
+ hyper.tls._context.verify_mode = hyper.compat.ssl.CERT_NONE
+
+ conn = HTTPConnection('localhost:{0}'.format(port), secure=True)
+
+ sites={'/'}
+ responses = []
+ request_ids = []
+ for site in sites:
+ request_id = conn.request('GET',url=site)
+ request_ids.append(request_id)
+
+ # get responses
+ for req_id in request_ids:
+ response = conn.get_response(req_id)
+ print(getResponseString(response))
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--port","-p",
+ type=int,
+ help="Port to use")
+ args=parser.parse_args()
+ makerequest(args.port)
+
+if __name__ == '__main__':
+ main()
+
diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py
new file mode 100644
index 0000000..2d81c1b
--- /dev/null
+++ b/tests/gold_tests/h2/http2.test.py
@@ -0,0 +1,84 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+Test.Summary = '''
+Test a basic remap of a http connection
+'''
+# need Curl
+Test.SkipUnless(
+ Condition.HasProgram("curl","Curl need to be installed on system for this test to work")
+ )
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts",select_ports=False)
+server=Test.MakeOriginServer("server")
+
+testName = ""
+request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+#desired response form the origin server
+response_header={"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+server.addResponse("sessionlog.json", request_header, response_header)
+
+# Add info for the large H2 download test
+server.addResponse("sessionlog.json",
+ {"headers": "GET /bigfile HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""},
+ {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 191414\r\n\r\n", "timestamp": "1469733493.993", "body": "" })
+
+
+#add ssl materials like key, certificates for the server
+ts.addSSLfile("ssl/server.pem")
+ts.addSSLfile("ssl/server.key")
+
+ts.Variables.ssl_port = 4443
+ts.Disk.remap_config.AddLine(
+ 'map / http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+ts.Disk.ssl_multicert_config.AddLine(
+ 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key'
+)
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 0,
+ 'proxy.config.diags.debug.tags': 'http',
+ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir),
+ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir),
+ 'proxy.config.ssl.number.threads': 0,
+ 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port,ts.Variables.ssl_port), # enable ssl port
+ 'proxy.config.ssl.client.verify.server': 0,
+ 'proxy.config.ssl.server.cipher_suite' : 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2',
+ })
+ts.Setup.CopyAs('h2client.py',Test.RunDirectory)
+ts.Setup.CopyAs('h2bigclient.py',Test.RunDirectory)
+
+# Test Case 1: basic H2 interaction
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='python3 h2client.py -p {0}'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(server)
+tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port))
+tr.Processes.Default.Streams.stdout="gold/remap-200.gold"
+tr.StillRunningAfter=server
+
+# Test Case 2: Make sure all the big file gets back. Regression test for issue 1646
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='python3 h2bigclient.py -p {0}'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stdout="gold/bigfile.gold"
+tr.StillRunningAfter=server
+
diff --git a/tests/gold_tests/h2/ssl/server.key b/tests/gold_tests/h2/ssl/server.key
new file mode 100644
index 0000000..4c7a661
--- /dev/null
+++ b/tests/gold_tests/h2/ssl/server.key
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E
+kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u
+SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB
+AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal
+B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv
+sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26
+GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe
+YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ
+pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q
+tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA
+yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML
+lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ
+vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z
+-----END RSA PRIVATE KEY-----
diff --git a/tests/gold_tests/h2/ssl/server.pem b/tests/gold_tests/h2/ssl/server.pem
new file mode 100644
index 0000000..a1de94f
--- /dev/null
+++ b/tests/gold_tests/h2/ssl/server.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E
+kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u
+SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB
+AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal
+B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv
+sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26
+GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe
+YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ
+pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q
+tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA
+yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML
+lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ
+vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICszCCAhwCCQCRJsJJ+mTsdDANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wHhcNMTYwODI1MjI1NzIxWhcNMTcwODI1MjI1NzIxWjCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11
+uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE
+lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1
+Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAXSVfZ5p1TkhW
+QiYq9nfQlBnX2NVaf8ymA8edQR0qH/QBv4/52bNNXC7V/V+ev9LCho2iRMeYYyXB
+yo1wBAGR83lS9cF/tOABcYrxjdP54Sfkyh5fomcg8SV7zap6C8mhbV8r3EujbKCx
+igH3fMX5F/eRwNCzaMMyQsXaxTJ3trk=
+-----END CERTIFICATE-----
diff --git a/tests/gold_tests/logging/custom-log.test.py b/tests/gold_tests/logging/custom-log.test.py
new file mode 100644
index 0000000..f58f146
--- /dev/null
+++ b/tests/gold_tests/logging/custom-log.test.py
@@ -0,0 +1,103 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+Test.Summary = '''
+Test custom log file format
+'''
+# need Curl
+Test.SkipUnless(
+ Condition.HasProgram(
+ "curl", "Curl need to be installed on system for this test to work"),
+ Condition.IsPlatform("linux")
+)
+
+# Define default ATS
+ts = Test.MakeATSProcess("ts")
+
+# setup some config file for this server
+ts.Disk.remap_config.AddLine(
+ 'map / http://www.linkedin.com/ @action=deny'
+)
+
+ts.Disk.logging_config.AddLines(
+ '''custom = format {
+ Format = "%<hii> %<hiih>"
+}
+
+log.ascii {
+ Format = custom,
+ Filename = 'test_log_field'
+}'''.split("\n")
+)
+
+# #########################################################################
+# at the end of the different test run a custom log file should exist
+# Because of this we expect the testruns to pass the real test is if the
+# customlog file exists and passes the format check
+Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'test_log_field.log'),
+ exists=True, content='gold/custom.gold')
+
+# first test is a miss for default
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(Test.Processes.ts)
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.1.1.1:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.2.2.2:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.3.3.3:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.3.0.1:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.43.2.1:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.213.213.132:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl "http://127.123.32.243:{0}" --verbose'.format(
+ ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+
+tr = Test.AddTestRun()
+tr.DelayStart=10
+tr.Processes.Default.Command = 'echo "Delay"'
+tr.Processes.Default.ReturnCode = 0
\ No newline at end of file
diff --git a/tests/gold_tests/logging/gold/custom.gold b/tests/gold_tests/logging/gold/custom.gold
new file mode 100644
index 0000000..4432d33
--- /dev/null
+++ b/tests/gold_tests/logging/gold/custom.gold
@@ -0,0 +1,8 @@
+127.0.0.1 7F000001
+127.1.1.1 7F010101
+127.2.2.2 7F020202
+127.3.3.3 7F030303
+127.3.0.1 7F030001
+127.43.2.1 7F2B0201
+127.213.213.132 7FD5D584
+127.123.32.243 7F7B20F3
diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-303.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-303.gold
new file mode 100644
index 0000000..94a42ad
--- /dev/null
+++ b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-303.gold
@@ -0,0 +1,14 @@
+``
+> GET http://www.example.com``
+> Host: www.example.com``
+> User-Agent: curl/``
+> Accept: */*
+``
+< HTTP/1.1 303 See Other
+< Date: ``
+< Age: ``
+< Transfer-Encoding: chunked
+< Proxy-Connection: keep-alive
+< Server: ATS/``
+<
+``
diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-tag.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-tag.gold
new file mode 100644
index 0000000..eac700f
--- /dev/null
+++ b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-tag.gold
@@ -0,0 +1 @@
+``DIAG: (header_rewrite)``
diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py
new file mode 100644
index 0000000..c9d6d73
--- /dev/null
+++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py
@@ -0,0 +1,68 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+Test.Summary = '''
+Test a basic remap of a http connection
+'''
+# need Curl
+Test.SkipUnless(
+ Condition.HasProgram("curl","Curl need to be installed on system for this test to work")
+ )
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts")
+server=Test.MakeOriginServer("server")
+
+Test.testName = ""
+request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+#expected response from the origin server
+response_header={"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+
+#add response to the server dictionary
+server.addResponse("sessionfile.log", request_header, response_header)
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'header.*',
+ })
+# The following rule changes the status code returned from origin server to 303
+ts.Setup.CopyAs('rules/rule.conf',Test.RunDirectory)
+ts.Disk.plugin_config.AddLine(
+ 'header_rewrite.so {0}/rule.conf'.format(Test.RunDirectory)
+)
+ts.Disk.remap_config.AddLine(
+ 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+ts.Disk.remap_config.AddLine(
+ 'map http://www.example.com:8080 http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+
+# call localhost straight
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: keep-alive" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(server,ready=When.PortOpen(server.Variables.Port))
+tr.Processes.Default.StartBefore(Test.Processes.ts)
+tr.Processes.Default.Streams.stderr="gold/header_rewrite-303.gold"
+tr.StillRunningAfter=server
+
+ts.Streams.All="gold/header_rewrite-tag.gold"
+
+
+
diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule.conf
new file mode 100644
index 0000000..551373b
--- /dev/null
+++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule.conf
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cond %{STATUS} =200
+set-status 303
diff --git a/tests/gold_tests/remap/gold/remap-200.gold b/tests/gold_tests/remap/gold/remap-200.gold
new file mode 100644
index 0000000..619066a
--- /dev/null
+++ b/tests/gold_tests/remap/gold/remap-200.gold
@@ -0,0 +1,14 @@
+``
+> GET http://www.example.com``
+> Host: www.example.com``
+> User-Agent: curl/``
+> Accept: */*
+``
+< HTTP/1.1 200 OK
+< Date: ``
+< Age: ``
+< Transfer-Encoding: chunked
+< Proxy-Connection: keep-alive
+< Server: ATS/``
+<
+``
diff --git a/tests/gold_tests/remap/gold/remap-404.gold b/tests/gold_tests/remap/gold/remap-404.gold
new file mode 100644
index 0000000..779053a
--- /dev/null
+++ b/tests/gold_tests/remap/gold/remap-404.gold
@@ -0,0 +1,12 @@
+``
+> GET `` HTTP/1.1
+> Host: ``
+> User-Agent: curl/``
+``
+< HTTP/1.1 404 Not Found
+< Date: ``
+< Proxy-Connection: keep-alive
+< Server: ATS/``
+``
+< Content-Type: text/html
+``
diff --git a/tests/gold_tests/remap/gold/remap-hitATS-404.gold b/tests/gold_tests/remap/gold/remap-hitATS-404.gold
new file mode 100644
index 0000000..67a81bc
--- /dev/null
+++ b/tests/gold_tests/remap/gold/remap-hitATS-404.gold
@@ -0,0 +1,11 @@
+``
+> GET / HTTP/1.1
+> Host: ``
+> User-Agent: curl/``
+``
+< HTTP/1.1 404 Not Found on Accelerator
+< Date: ``
+< Connection: ``
+< Server: ATS/``
+< Content-Type: text/html
+``
diff --git a/tests/gold_tests/remap/gold/remap-https-200.gold b/tests/gold_tests/remap/gold/remap-https-200.gold
new file mode 100644
index 0000000..9cd42fc
--- /dev/null
+++ b/tests/gold_tests/remap/gold/remap-https-200.gold
@@ -0,0 +1,13 @@
+``
+> GET / HTTP/1.1
+> Host: www.example.com``
+> User-Agent: curl/``
+``
+< HTTP/1.1 200 OK
+< Date: ``
+< Age: ``
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Server: ATS/``
+<
+``
diff --git a/tests/gold_tests/remap/remap_http.test.py b/tests/gold_tests/remap/remap_http.test.py
new file mode 100644
index 0000000..a6fa9cb
--- /dev/null
+++ b/tests/gold_tests/remap/remap_http.test.py
@@ -0,0 +1,91 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+Test.Summary = '''
+Test a basic remap of a http connection
+'''
+# need Curl
+Test.SkipUnless(
+ Condition.HasProgram("curl","Curl need to be installed on system for this test to work")
+ )
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts")
+server=Test.MakeOriginServer("server")
+
+Test.testName = ""
+request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+#expected response from the origin server
+response_header={"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+
+#add response to the server dictionary
+server.addResponse("sessionfile.log", request_header, response_header)
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'url.*',
+ })
+
+ts.Disk.remap_config.AddLine(
+ 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+ts.Disk.remap_config.AddLine(
+ 'map http://www.example.com:8080 http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+
+# call localhost straight
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl "http://127.0.0.1:{0}/" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(server)
+tr.Processes.Default.StartBefore(Test.Processes.ts)
+tr.Processes.Default.Streams.stderr="gold/remap-hitATS-404.gold"
+tr.StillRunningAfter=server
+
+# www.example.com host
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: keep-alive" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-200.gold"
+
+# www.example.com:80 host
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.example.com:80/" -H "Proxy-Connection: keep-alive" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-200.gold"
+
+# www.example.com:8080 host
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.example.com:8080" -H "Proxy-Connection: keep-alive" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-200.gold"
+
+# no rule for this
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.test.com/" -H "Proxy-Connection: keep-alive" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-404.gold"
+
+# bad port
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.example.com:1234/" -H "Proxy-Connection: keep-alive" --verbose'.format(ts.Variables.port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-404.gold"
+
+
diff --git a/tests/gold_tests/remap/remap_https.test.py b/tests/gold_tests/remap/remap_https.test.py
new file mode 100644
index 0000000..7c74478
--- /dev/null
+++ b/tests/gold_tests/remap/remap_https.test.py
@@ -0,0 +1,112 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+Test.Summary = '''
+Test a basic remap of a http connection
+'''
+# need Curl
+Test.SkipUnless(
+ Condition.HasProgram("curl","Curl need to be installed on system for this test to work")
+ )
+Test.ContinueOnFail=True
+# Define default ATS
+ts=Test.MakeATSProcess("ts",select_ports=False)
+server=Test.MakeOriginServer("server")
+
+#**testname is required**
+testName = ""
+request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+#desired response form the origin server
+response_header={"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+server.addResponse("sessionlog.json", request_header, response_header)
+
+#add ssl materials like key, certificates for the server
+ts.addSSLfile("ssl/server.pem")
+ts.addSSLfile("ssl/server.key")
+
+ts.Variables.ssl_port = 4443
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'lm|ssl',
+ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir),
+ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir),
+ 'proxy.config.ssl.number.threads': 0,
+ 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port,ts.Variables.ssl_port), # enable ssl port
+ 'proxy.config.ssl.client.verify.server': 0,
+ 'proxy.config.ssl.server.cipher_suite' : 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2',
+ })
+
+ts.Disk.remap_config.AddLine(
+ 'map https://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+ts.Disk.remap_config.AddLine(
+ 'map https://www.example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port)
+)
+
+
+ts.Disk.ssl_multicert_config.AddLine(
+ 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key'
+)
+
+# call localhost straight
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --http1.1 -k https://127.0.0.1:{0} --verbose'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+
+# time delay as proxy.config.http.wait_for_cache could be broken
+tr.Processes.Default.StartBefore(server)
+# Delay on readyness of our ssl ports
+tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port))
+tr.Processes.Default.Streams.stderr="gold/remap-hitATS-404.gold"
+tr.StillRunningAfter=server
+tr.StillRunningAfter=ts
+
+
+# www.example.com host
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --http1.1 -k https://127.0.0.1:{0} -H "Host: www.example.com" --verbose'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-https-200.gold"
+
+
+# www.example.com:80 host
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --http1.1 -k https://127.0.0.1:{0} -H "Host: www.example.com:443" --verbose'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-https-200.gold"
+
+# www.example.com:8080 host
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --http1.1 -k https://127.0.0.1:{0} -H "Host: www.example.com:4443" --verbose'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-https-200.gold"
+
+# no rule for this
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --http1.1 -k https://127.0.0.1:{0} -H "Host: www.test.com" --verbose'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-hitATS-404.gold"
+
+# bad port
+tr=Test.AddTestRun()
+tr.Processes.Default.Command='curl --http1.1 -k https://127.0.0.1:{0} -H "Host: www.example.com:1234" --verbose'.format(ts.Variables.ssl_port)
+tr.Processes.Default.ReturnCode=0
+tr.Processes.Default.Streams.stderr="gold/remap-hitATS-404.gold"
+
+
diff --git a/tests/gold_tests/remap/ssl/server.key b/tests/gold_tests/remap/ssl/server.key
new file mode 100644
index 0000000..4c7a661
--- /dev/null
+++ b/tests/gold_tests/remap/ssl/server.key
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E
+kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u
+SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB
+AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal
+B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv
+sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26
+GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe
+YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ
+pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q
+tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA
+yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML
+lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ
+vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z
+-----END RSA PRIVATE KEY-----
diff --git a/tests/gold_tests/remap/ssl/server.pem b/tests/gold_tests/remap/ssl/server.pem
new file mode 100644
index 0000000..a1de94f
--- /dev/null
+++ b/tests/gold_tests/remap/ssl/server.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E
+kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u
+SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB
+AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal
+B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv
+sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26
+GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe
+YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ
+pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q
+tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA
+yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML
+lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ
+vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICszCCAhwCCQCRJsJJ+mTsdDANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wHhcNMTYwODI1MjI1NzIxWhcNMTcwODI1MjI1NzIxWjCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11
+uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE
+lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1
+Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAXSVfZ5p1TkhW
+QiYq9nfQlBnX2NVaf8ymA8edQR0qH/QBv4/52bNNXC7V/V+ev9LCho2iRMeYYyXB
+yo1wBAGR83lS9cF/tOABcYrxjdP54Sfkyh5fomcg8SV7zap6C8mhbV8r3EujbKCx
+igH3fMX5F/eRwNCzaMMyQsXaxTJ3trk=
+-----END CERTIFICATE-----
diff --git a/tests/tools/README.md b/tests/tools/README.md
new file mode 100644
index 0000000..d6d17c0
--- /dev/null
+++ b/tests/tools/README.md
@@ -0,0 +1,42 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+These tools are meant to become it own repository in the future. They are here at the moment to help accelerate progress at getting everything working.
+
+Note these Tools require python 3.4 or better.
+
+# Traffic-Replay
+
+Replay client to replay session logs.
+
+Usage:
+python3.5 trafficreplay_v2/ -type <ssl|h2|random> -log_dir /path/to/log -v
+
+Session Log format (in JSON):
+
+ {"version": "0.1",
+ "txns": [
+ {"request": {"headers": "POST \u2026\u2026\r\n\r\n", "timestamp": "..", "body": ".."},
+ "response": {"headers": "HTTP/1.1..\r\n\r\n", "timestamp": "..", "body": ".."},
+ "uuid": "1"},
+ {"request": {"headers": "POST ..\u2026.\r\n\r\n", "timestamp": "..", "body": ".."},
+ "response": {"headers": "HTTP/1.1..\r\nr\n", "timestamp": "..", "body": ".."},
+ "uuid": "2"}
+ ],
+ "timestamp": "....",
+ "encoding": "...."}
+ Configuration: The configuration required to run traffic-replay can be specified in traffic-replay/Config.py
diff --git a/tests/tools/lib/result.py b/tests/tools/lib/result.py
new file mode 100644
index 0000000..992879b
--- /dev/null
+++ b/tests/tools/lib/result.py
@@ -0,0 +1,91 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+class TermColors:
+ ''' Collection of colors for printing out to terminal '''
+ HEADER = '\033[95m'
+ OKBLUE = '\033[94m'
+ OKGREEN = '\033[92m'
+ WARNING = '\033[93m'
+ FAIL = '\033[91m'
+ BOLD = '\033[1m'
+ UNDERLINE = '\033[4m'
+ ENDC = '\033[0m'
+
+ignoredFields = {'age', 'set-cookie', 'server', 'date', 'last-modified', 'via', 'expires', 'cahe-control'} #all lower case
+class Result(object):
+ ''' Result encapsulates the result of a single session replay '''
+
+ def __init__(self, test_name, expected_response, received_response, recv_resp_body=None):
+ ''' expected_response and received_response can be any datatype the caller wants as long as they are the same datatype '''
+ self._test_name = test_name
+ self._expected_response = expected_response
+ self._received_response = received_response
+ self._received_response_body = recv_resp_body
+
+ def getTestName(self):
+ return self._test_name
+
+ def getResultBool(self):
+ return self._expected_response == self._received_response
+
+ def getRespBody(self):
+ if self._received_response_body:
+ return self._received_response_body
+ else:
+ return ""
+ def Compare(self, received_dict, expected_dict):
+ global ignoredFields
+ try:
+ for key in received_dict:
+ #print(key)
+ if key.lower() in expected_dict and key.lower() not in ignoredFields:
+ #print("{0} ==? {1}".format(expected_dict[key.lower()],received_dict[key]))
+ if received_dict[key]!=expected_dict[key.lower()]:
+ print("{0}Difference in the field \"{1}\": \n received:\n{2}\n expected:\n{3}{4}".format(TermColors.FAIL,key,received_dict[key],expected_dict[key],TermColors.ENDC))
+ return False
+
+ except:
+ e=sys.exc_info()
+ print("Error in comparing key ",e,key,expected_dict[key.lower()],received_dict[key])
+ return True
+
+ def getResultString(self, received_dict, expected_dict, colorize=False ):
+ global ignoredFields
+ ''' Return a nicely formatted result string with color if requested '''
+ if self.getResultBool() and self.Compare(received_dict,expected_dict):
+ if colorize:
+ outstr = "{0}PASS{1}".format(
+ TermColors.OKGREEN, TermColors.ENDC)
+
+ else:
+ outstr = "PASS"
+
+ else:
+ if colorize:
+ outstr = "{0}FAIL{1}: expected {2}, received {3}, session file: {4}".format(
+ TermColors.FAIL, TermColors.ENDC, self._expected_response, self._received_response, self._test_name)
+
+ else:
+ outstr = "FAIL: expected {0}, received {1}".format(
+ self._expected_response, self._received_response)
+ sys.exit(0)
+
+ return outstr
diff --git a/tests/tools/microServer/ssl/server.crt b/tests/tools/microServer/ssl/server.crt
new file mode 100644
index 0000000..0ce6ac5
--- /dev/null
+++ b/tests/tools/microServer/ssl/server.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICszCCAhwCCQCRJsJJ+mTsdDANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wHhcNMTYwODI1MjI1NzIxWhcNMTcwODI1MjI1NzIxWjCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11
+uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE
+lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1
+Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAXSVfZ5p1TkhW
+QiYq9nfQlBnX2NVaf8ymA8edQR0qH/QBv4/52bNNXC7V/V+ev9LCho2iRMeYYyXB
+yo1wBAGR83lS9cF/tOABcYrxjdP54Sfkyh5fomcg8SV7zap6C8mhbV8r3EujbKCx
+igH3fMX5F/eRwNCzaMMyQsXaxTJ3trk=
+-----END CERTIFICATE-----
diff --git a/tests/tools/microServer/ssl/server.pem b/tests/tools/microServer/ssl/server.pem
new file mode 100644
index 0000000..a1de94f
--- /dev/null
+++ b/tests/tools/microServer/ssl/server.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E
+kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u
+SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB
+AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal
+B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv
+sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26
+GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe
+YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ
+pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q
+tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA
+yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML
+lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ
+vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICszCCAhwCCQCRJsJJ+mTsdDANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wHhcNMTYwODI1MjI1NzIxWhcNMTcwODI1MjI1NzIxWjCBnTELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDjAMBgNVBAoMBVlh
+aG9vMQ0wCwYDVQQLDARFZGdlMSgwJgYDVQQDDB9qdWljZXByb2R1Y2UuY29ycC5u
+ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j
+b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11
+uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE
+lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1
+Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAXSVfZ5p1TkhW
+QiYq9nfQlBnX2NVaf8ymA8edQR0qH/QBv4/52bNNXC7V/V+ev9LCho2iRMeYYyXB
+yo1wBAGR83lS9cF/tOABcYrxjdP54Sfkyh5fomcg8SV7zap6C8mhbV8r3EujbKCx
+igH3fMX5F/eRwNCzaMMyQsXaxTJ3trk=
+-----END CERTIFICATE-----
diff --git a/tests/tools/microServer/uWServer.py b/tests/tools/microServer/uWServer.py
new file mode 100644
index 0000000..84ed27a
--- /dev/null
+++ b/tests/tools/microServer/uWServer.py
@@ -0,0 +1,676 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import string
+import http.client
+import cgi
+import time
+import sys
+import json
+import os
+import threading
+from ipaddress import ip_address
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from socketserver import ThreadingMixIn, ForkingMixIn, BaseServer
+from http import HTTPStatus
+import argparse
+import ssl
+import socket
+import importlib.util
+
+test_mode_enabled = True
+__version__="1.0"
+
+
+sys.path.append(
+ os.path.normpath(
+ os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ '..'
+ )
+ )
+ )
+
+import sessionvalidation.sessionvalidation as sv
+
+
+SERVER_PORT = 5005 # default port
+HTTP_VERSION = 'HTTP/1.1'
+G_replay_dict = {}
+
+count = 0
+
+# Simple class to hold lists of callbacks associated with a key.
+class HookSet:
+ # Helper class to provide controlled access to the HookSet to the loading module.
+ class Registrar :
+ def __init__(self, hook_set) :
+ self.hooks = hook_set
+
+ def register(self, hook, cb) :
+ self.hooks.register(hook, cb)
+
+ def __init__(self) :
+ self.hooks = {}
+ self.modules = []
+ self.registrar = HookSet.Registrar(self)
+ # Define all the valid hooks here.
+ for item in [ 'ReadRequestHook' ] :
+ if isinstance(item, list) :
+ hook = item[0];
+ label = item[1];
+ else :
+ hook = label = item
+ exec("HookSet.{} = '{}'".format(label, hook))
+ exec("HookSet.Registrar.{} = '{}'".format(label, hook))
+ self.hooks[hook] = []
+
+ def load(self, source) :
+ try :
+ spec = importlib.util.spec_from_file_location('Observer', source)
+ mod = importlib.util.module_from_spec(spec)
+ mod.Hooks = self.registrar
+ spec.loader.exec_module(mod)
+ except ImportError:
+ print("Failed to import {}".format(source))
+ else :
+ self.modules.append(mod)
+
+ # Add a callback cb to the hook.
+ # Error if the hook isn't defined.
+ def register(self, hook, cb):
+ if hook in self.hooks :
+ self.hooks[hook].append(cb)
+ else:
+ raise ValueError("{} is not a valid hook name".format(hook))
+
+ # Invoke a hook. Pass on any additional arguments to the callback.
+ def invoke(self, hook, *args, **kwargs):
+ cb_list = self.hooks[hook]
+ if cb_list == None :
+ raise ValueError("{} is not a valid hook name to invoke".format(hook))
+ else :
+ for cb in cb_list :
+ cb(*args, **kwargs)
+
+class ThreadingServer(ThreadingMixIn, HTTPServer):
+ '''This class forces the creation of a new thread on each connection'''
+ def __init__(self, local_addr, handler_class, options) :
+ HTTPServer.__init__(self, local_addr, handler_class)
+ self.hook_set = HookSet()
+ if (options.load) :
+ self.hook_set.load(options.load)
+
+class ForkingServer(ForkingMixIn, HTTPServer):
+ '''This class forces the creation of a new process on each connection'''
+ pass
+
+class SSLServer(ThreadingMixIn, HTTPServer):
+ def __init__(self, server_address, HandlerClass, options):
+ BaseServer.__init__(self, server_address, HandlerClass)
+ pwd = os.path.dirname(os.path.realpath(__file__))
+ keys = os.path.join(pwd,options.key)
+ certs = os.path.join(pwd,options.cert)
+ self.options = options
+ self.hook_set = HookSet()
+
+ self.daemon_threads = True
+ self.protocol_version = 'HTTP/1.1'
+
+ if options.load :
+ self.hook_set.load(options.load)
+
+ if options.clientverify:
+ self.socket = ssl.wrap_socket(socket.socket(self.address_family, self.socket_type),
+ keyfile=keys, certfile=certs, server_side=True, cert_reqs=ssl.CERT_REQUIRED, ca_certs='/etc/ssl/certs/ca-certificates.crt')
+ else:
+ self.socket = ssl.wrap_socket(socket.socket(self.address_family, self.socket_type),
+ keyfile=keys, certfile=certs, server_side=True)
+
+ self.server_bind()
+ self.server_activate()
+
+
+class MyHandler(BaseHTTPRequestHandler):
+ def handleExpect100Continue(self,contentLength,chunked=False):
+ print("....expect",contentLength)
+ self.wfile.write(bytes('HTTP/1.1 100 Continue\r\n\r\n','UTF-8'))
+ #self.send_response(HTTPStatus.CONTINUE)
+ #self.send_header('Server','blablabla')
+ #self.send_header('Connection', 'keep-alive')
+ #self.end_headers()
+ if(not chunked):
+ message = self.rfile.read(contentLength)
+ else:
+ readChunks()
+
+ def getTestName(self,requestline):
+ key=None
+ keys=requestline.split(" ")
+ #print(keys)
+ if keys:
+ rkey=keys[1]
+ key=rkey.split("/",1)[1]
+ if key+"/" in G_replay_dict:
+ key = key+"/"
+ elif len(key) > 1 and key[:-1] in G_replay_dict:
+ key = key[:-1]
+ return key
+
+ def parseRequestline(self,requestline):
+ testName=None
+ return testName
+
+ def testMode(self,requestline):
+ print(requestline)
+ key=self.parseRequestline(requestline)
+
+ self.send_response(200)
+ self.send_header('Connection', 'close')
+ self.end_headers()
+
+
+ def get_response_code(self, header):
+ # this could totally go wrong
+ return int(header.split(' ')[1])
+
+ def generator(self):
+ yield 'microserver'
+ yield 'yahoo'
+ def send_response(self, code, message=None):
+ ''' Override `send_response()`'s tacking on of server and date header lines. '''
+ #self.log_request(code)
+ self.send_response_only(code, message)
+
+ def createDummyBodywithLength(self,numberOfbytes):
+ if numberOfbytes==0:
+ return None
+ body= 'a'
+ while numberOfbytes!=1:
+ body += 'b'
+ numberOfbytes -= 1
+ return body
+
+ def writeChunkedData(self):
+ for chunk in self.generator():
+ response_string=bytes('%X\r\n%s\r\n'%(len(chunk),chunk),'UTF-8')
+ self.wfile.write(response_string)
+ response_string=bytes('0\r\n\r\n','UTF-8')
+ self.wfile.write(response_string)
+
+ def readChunks(self):
+ raw_data=b''
+ raw_size = self.rfile.readline(65537)
+ size = str(raw_size, 'UTF-8').rstrip('\r\n')
+ #print("==========================================>",size)
+ size = int(size,16)
+ while size>0:
+ #print("reading bytes",raw_size)
+ chunk = self.rfile.read(size+2) # 2 for reading /r/n
+ #print("cuhnk: ",chunk)
+ raw_data += chunk
+ raw_size = self.rfile.readline(65537)
+ size = str(raw_size, 'UTF-8').rstrip('\r\n')
+ size = int(size,16)
+ #print("full chunk",raw_data)
+ chunk = self.rfile.readline(65537) # read the extra blank newline \r\n after the last chunk
+
+ def send_header(self, keyword, value):
+ """Send a MIME header to the headers buffer."""
+ if self.request_version != 'HTTP/0.9':
+ if not hasattr(self, '_headers_buffer'):
+ self._headers_buffer = []
+ self._headers_buffer.append(
+ ("%s: %s\r\n" % (keyword, value)).encode('UTF-8', 'strict')) #original code used latin-1.. seriously?
+
+ if keyword.lower() == 'connection':
+ if value.lower() == 'close':
+ self.close_connection = True
+ elif value.lower() == 'keep-alive':
+ self.close_connection = False
+ def parse_request(self):
+ """Parse a request (internal).
+
+ The request should be stored in self.raw_requestline; the results
+ are in self.command, self.path, self.request_version and
+ self.headers.
+
+ Return True for success, False for failure; on failure, an
+ error is sent back.
+
+ """
+
+ global count, test_mode_enabled
+
+ self.command = None # set in case of error on the first line
+ self.request_version = version = self.default_request_version
+ self.close_connection = True
+ requestline = str(self.raw_requestline, 'UTF-8')
+ #print("request",requestline)
+ requestline = requestline.rstrip('\r\n')
+ self.requestline = requestline
+
+ # Examine the headers and look for a Connection directive.
+ try:
+ self.headers = http.client.parse_headers(self.rfile,
+ _class=self.MessageClass)
+ self.server.hook_set.invoke(HookSet.ReadRequestHook, self.headers)
+
+ # read message body
+ if self.headers.get('Content-Length') != None:
+ bodysize = int(self.headers.get('Content-Length'))
+ #print("length of the body is",bodysize)
+ message = self.rfile.read(bodysize)
+ #print("message body",message)
+ elif self.headers.get('Transfer-Encoding',"") == 'chunked':
+ #print(self.headers)
+ self.readChunks()
+ except http.client.LineTooLong:
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Line too long")
+ return False
+ except http.client.HTTPException as err:
+ self.send_error(
+ HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE,
+ "Too many headers",
+ str(err)
+ )
+ return False
+
+
+ words = requestline.split()
+ if len(words) == 3:
+ command, path, version = words
+ if version[:5] != 'HTTP/':
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad request version (%r)" % version)
+ return False
+ try:
+ base_version_number = version.split('/', 1)[1]
+ version_number = base_version_number.split(".")
+ # RFC 2145 section 3.1 says there can be only one "." and
+ # - major and minor numbers MUST be treated as
+ # separate integers;
+ # - HTTP/2.4 is a lower version than HTTP/2.13, which in
+ # turn is lower than HTTP/12.3;
+ # - Leading zeros MUST be ignored by recipients.
+ if len(version_number) != 2:
+ raise ValueError
+ version_number = int(version_number[0]), int(version_number[1])
+ except (ValueError, IndexError):
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad request version (%r)" % version)
+ return False
+ if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1":
+ self.close_connection = False
+ if version_number >= (2, 0):
+ self.send_error(
+ HTTPStatus.HTTP_VERSION_NOT_SUPPORTED,
+ "Invalid HTTP Version (%s)" % base_version_number)
+ return False
+ elif len(words) == 2:
+ command, path = words
+ self.close_connection = True
+ if command != 'GET':
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad HTTP/0.9 request type (%r)" % command)
+ return False
+ elif not words:
+ count += 1
+ print("bla bla on 157 {0} => {1}".format(count,self.close_connection))
+ return False
+ else:
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad request syntax (%r)" % requestline)
+ return False
+ self.command, self.path, self.request_version = command, path, version
+
+ conntype = self.headers.get('Connection', "")
+ if conntype.lower() == 'close':
+ self.close_connection = True
+ elif (conntype.lower() == 'keep-alive' and
+ self.protocol_version >= "HTTP/1.1"):
+ self.close_connection = False
+
+ return True
+
+ def do_GET(self):
+ global G_replay_dict, test_mode_enabled
+ if test_mode_enabled:
+ request_hash = self.getTestName(self.requestline)
+ else:
+ request_hash, __ = cgi.parse_header(self.headers.get('Content-MD5'))
+ #print("key:",request_hash)
+ try:
+ response_string=None
+ chunkedResponse= False
+ if request_hash not in G_replay_dict:
+ self.send_response(404)
+ self.send_header('Server','MicroServer')
+ self.send_header('Connection', 'close')
+ self.end_headers()
+
+ else:
+ resp = G_replay_dict[request_hash]
+ headers = resp.getHeaders().split('\r\n')
+
+ # set status codes
+ status_code = self.get_response_code(headers[0])
+ self.send_response(status_code)
+
+ # set headers
+ for header in headers[1:]: # skip first one b/c it's response code
+ if header == '':
+ continue
+ elif 'Content-Length' in header:
+ if 'Access-Control' in header: # skipping Access-Control-Allow-Credentials, Access-Control-Allow-Origin, Content-Length
+ header_parts = header.split(':', 1)
+ header_field = str(header_parts[0].strip())
+ header_field_val = str(header_parts[1].strip())
+ self.send_header(header_field, header_field_val)
+ continue
+ lengthSTR = header.split(':')[1]
+ length = lengthSTR.strip(' ')
+ if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. But in replay mode Content-Length gets the priority
+ if not (resp and resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester
+ self.send_header('Content-Length', str(length))
+ else:
+ self.send_header('Content-Length', str(length))
+ response_string = self.createDummyBodywithLength(int(length))
+ continue
+ if 'Transfer-Encoding' in header:
+ self.send_header('Transfer-Encoding','Chunked')
+ response_string='%X\r\n%s\r\n'%(len('ats'),'ats')
+ chunkedResponse= True
+ continue
+
+ header_parts = header.split(':', 1)
+ header_field = str(header_parts[0].strip())
+ header_field_val = str(header_parts[1].strip())
+ #print("{0} === >{1}".format(header_field, header_field_val))
+ self.send_header(header_field, header_field_val)
+ #End for
+ if test_mode_enabled:
+ if resp and resp.getBody():
+ length = len(bytes(resp.getBody(),'UTF-8'))
+ response_string=resp.getBody()
+ self.send_header('Content-Length', str(length))
+ self.end_headers()
+
+
+ if (chunkedResponse):
+ self.writeChunkedData()
+ elif response_string!=None and response_string!='':
+ self.wfile.write(bytes(response_string, 'UTF-8'))
+ return
+ except:
+ e=sys.exc_info()
+ print("Error",e,self.headers)
+ self.send_response(400)
+ self.send_header('Connection', 'close')
+ self.end_headers()
+
+
+
+ def do_HEAD(self):
+ global G_replay_dict, test_mode_enabled
+ if test_mode_enabled:
+ request_hash = self.getTestName(self.requestline)
+ else:
+ request_hash, __ = cgi.parse_header(self.headers.get('Content-MD5'))
+
+ if request_hash not in G_replay_dict:
+ self.send_response(404)
+ self.send_header('Connection', 'close')
+ self.end_headers()
+
+ else:
+ resp = G_replay_dict[request_hash]
+ headers = resp.getHeaders().split('\r\n')
+
+ # set status codes
+ status_code = self.get_response_code(headers[0])
+ self.send_response(status_code)
+
+ # set headers
+ for header in headers[1:]: # skip first one b/c it's response code
+ if header == '':
+ continue
+ elif 'Content-Length' in header:
+ self.send_header('Content-Length', '0')
+ continue
+
+ header_parts = header.split(':', 1)
+ header_field = str(header_parts[0].strip())
+ header_field_val = str(header_parts[1].strip())
+ #print("{0} === >{1}".format(header_field, header_field_val))
+ self.send_header(header_field, header_field_val)
+
+ self.end_headers()
+
+ def do_POST(self):
+ response_string=None
+ chunkedResponse= False
+ global G_replay_dict, test_mode_enabled
+ if test_mode_enabled:
+ request_hash = self.getTestName(self.requestline)
+ else:
+ request_hash, __ = cgi.parse_header(self.headers.get('Content-MD5'))
+ try:
+
+ if request_hash not in G_replay_dict:
+ self.send_response(404)
+ self.send_header('Connection', 'close')
+ self.end_headers()
+ resp = None
+ else:
+ resp = G_replay_dict[request_hash]
+ resp_headers = resp.getHeaders().split('\r\n')
+ # set status codes
+ status_code = self.get_response_code(resp_headers[0])
+ #print("response code",status_code)
+ self.send_response(status_code)
+ #print("reposen is ",resp_headers)
+ # set headers
+ for header in resp_headers[1:]: # skip first one b/c it's response code
+
+ if header == '':
+ continue
+ elif 'Content-Length' in header:
+ if 'Access-Control' in header: # skipping Access-Control-Allow-Credentials, Access-Control-Allow-Origin, Content-Length
+ header_parts = header.split(':', 1)
+ header_field = str(header_parts[0].strip())
+ header_field_val = str(header_parts[1].strip())
+ self.send_header(header_field, header_field_val)
+ continue
+
+ lengthSTR = header.split(':')[1]
+ length = lengthSTR.strip(' ')
+ if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. But in replay mode Content-Length gets the priority
+ if not (resp and resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester
+ self.send_header('Content-Length', str(length))
+ else:
+ self.send_header('Content-Length', str(length))
+ response_string = self.createDummyBodywithLength(int(length))
+ continue
+ if 'Transfer-Encoding' in header:
+ self.send_header('Transfer-Encoding','Chunked')
+ response_string='%X\r\n%s\r\n'%(len('microserver'),'microserver')
+ chunkedResponse= True
+ continue
+
+ header_parts = header.split(':', 1)
+ header_field = str(header_parts[0].strip())
+ header_field_val = str(header_parts[1].strip())
+ #print("{0} === >{1}".format(header_field, header_field_val))
+ self.send_header(header_field, header_field_val)
+ # End for loop
+ if test_mode_enabled:
+ if resp and resp.getBody():
+ length = len(bytes(resp.getBody(),'UTF-8'))
+ response_string=resp.getBody()
+ self.send_header('Content-Length', str(length))
+ self.end_headers()
+
+ if (chunkedResponse):
+ self.writeChunkedData()
+ elif response_string!=None and response_string!='':
+ self.wfile.write(bytes(response_string, 'UTF-8'))
+ return
+ except:
+ e=sys.exc_info()
+ print("Error",e,self.headers)
+ self.send_response(400)
+ self.send_header('Connection', 'close')
+ self.end_headers()
+
+def populate_global_replay_dictionary(sessions):
+ ''' Populates the global dictionary of {uuid (string): reponse (Response object)} '''
+ global G_replay_dict
+ for session in sessions:
+ for txn in session.getTransactionIter():
+ G_replay_dict[txn._uuid] = txn.getResponse()
+
+ print("size",len(G_replay_dict))
+
+#tests will add responses to the dictionary where key is the testname
+def addResponseHeader(key,response_header):
+ G_replay_dict[key] = response_header
+
+def _path(exists, arg ):
+ path = os.path.abspath(arg)
+ if not os.path.exists(path) and exists:
+ msg = '"{0}" is not a valid path'.format(path)
+ raise argparse.ArgumentTypeError(msg)
+ return path
+
+def _bool(arg):
+
+ opt_true_values = set(['y', 'yes', 'true', 't', '1', 'on' , 'all'])
+ opt_false_values = set(['n', 'no', 'false', 'f', '0', 'off', 'none'])
+
+ tmp = arg.lower()
+ if tmp in opt_true_values:
+ return True
+ elif tmp in opt_false_values:
+ return False
+ else:
+ msg = 'Invalid value Boolean value : "{0}"\n Valid options are {0}'.format(arg,
+ opt_true_values | opt_false_values)
+ raise argparse.ArgumentTypeError(msg)
+
+
+def main():
+ global test_mode_enabled
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("--data-dir","-d",
+ type=lambda x: _path(True,x),
+ required=True,
+ help="Directory with data file"
+ )
+
+ parser.add_argument("--public","-P",
+ type=_bool,
+ default=False,
+ help="Bind server to public IP 0.0.0.0 vs private IP of 127.0.0.1"
+ )
+
+ parser.add_argument("--ip_address","-ip",
+ type=str,
+ default='',
+ help="IP address of the interface to serve on"
+ )
+
+ parser.add_argument("--port","-p",
+ type=int,
+ default=SERVER_PORT,
+ help="Port to use")
+
+ parser.add_argument("--timeout","-t",
+ type=float,
+ default=None,
+ help="socket time out in seconds")
+
+ parser.add_argument('-V','--version', action='version', version='%(prog)s {0}'.format(__version__))
+
+ parser.add_argument("--mode","-m",
+ type=str,
+ default="test",
+ help="Mode of operation")
+ parser.add_argument("--connection","-c",
+ type=str,
+ default="nonSSL",
+ help="use SSL")
+ parser.add_argument("--key","-k",
+ type=str,
+ default="ssl/server.pem",
+ help="key for ssl connnection")
+ parser.add_argument("--cert","-cert",
+ type=str,
+ default="ssl/server.crt",
+ help="certificate")
+ parser.add_argument("--clientverify","-cverify",
+ type=bool,
+ default=False,
+ help="verify client cert")
+ parser.add_argument("--load",
+ dest='load',
+ type=str,
+ default='',
+ help="A file which will install observers on hooks")
+
+ args=parser.parse_args()
+ options = args
+
+ # set up global dictionary of {uuid (string): response (Response object)}
+ s = sv.SessionValidator(args.data_dir)
+ populate_global_replay_dictionary(s.getSessionIter())
+ print("Dropped {0} sessions for being malformed".format(len(s.getBadSessionList())))
+
+ # start server
+ try:
+ socket_timeout = args.timeout
+ test_mode_enabled = args.mode=="test"
+
+ MyHandler.protocol_version = HTTP_VERSION
+ if options.connection == 'ssl':
+ server = SSLServer((options.ip_address,options.port), MyHandler, options)
+ else:
+ server = ThreadingServer((options.ip_address, options.port), MyHandler, options)
+ server.timeout = 5
+ print("started server")
+ server_thread = threading.Thread(target=server.serve_forever())
+ server_thread.daemon=True
+ server_thread.start()
+
+ except KeyboardInterrupt:
+ print("\n=== ^C received, shutting down httpserver ===")
+ server.socket.close()
+ #s_server.socket.close()
+ sys.exit(0)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/tools/sessionvalidation/__init__.py b/tests/tools/sessionvalidation/__init__.py
new file mode 100644
index 0000000..bcbf685
--- /dev/null
+++ b/tests/tools/sessionvalidation/__init__.py
@@ -0,0 +1,17 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/tools/sessionvalidation/badsession.py b/tests/tools/sessionvalidation/badsession.py
new file mode 100644
index 0000000..50e0047
--- /dev/null
+++ b/tests/tools/sessionvalidation/badsession.py
@@ -0,0 +1,33 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class BadSession(object):
+ '''
+ Session encapsulates a single BAD user session. Bad meaning that for some reason the session is invalid.
+
+ _filename is the filename of the bad JSON session
+ _reason is a string with some kind of explanation on why the session was bad
+ '''
+
+ def __repr__(self):
+ return "<Session {{'filename': {0}, 'reason': {1}>".format(
+ self._filename, self._reason
+ )
+
+ def __init__(self, filename, reason):
+ self._filename = filename
+ self._reason = reason
diff --git a/tests/tools/sessionvalidation/request.py b/tests/tools/sessionvalidation/request.py
new file mode 100644
index 0000000..e664642
--- /dev/null
+++ b/tests/tools/sessionvalidation/request.py
@@ -0,0 +1,47 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+
+class Request(object):
+ ''' Request encapsulates a single request from the UA '''
+
+ def getTimestamp(self):
+ return self._timestamp
+
+ def getHeaders(self):
+ return self._headers
+
+ def getBody(self):
+ return self._body
+
+ def getHeaderMD5(self):
+ ''' Returns the MD5 hash of the headers
+
+ This is used to do a unique mapping to a request/response transaction '''
+ return hashlib.md5(self._headers.encode()).hexdigest()
+
+ def __repr__(self):
+ #return str(self._timestamp)
+ return "<Request: {{'timestamp': {0}, 'headers': {1}, 'body': {2}}}>".format(
+ str(self._timestamp), str(self._headers), str(self._body)
+ )
+
+ def __init__(self, timestamp, headers, body):
+ self._timestamp = timestamp
+ self._headers = headers
+ self._body = body
diff --git a/tests/tools/sessionvalidation/response.py b/tests/tools/sessionvalidation/response.py
new file mode 100644
index 0000000..49b0d02
--- /dev/null
+++ b/tests/tools/sessionvalidation/response.py
@@ -0,0 +1,38 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class Response(object):
+ ''' Response encapsulates a single request from the UA '''
+
+ def getTimestamp(self):
+ return self._timestamp
+
+ def getHeaders(self):
+ return self._headers
+
+ def getBody(self):
+ return self._body
+
+ def __repr__(self):
+ return "<Response: {{'timestamp': {0}, 'headers': {1}, 'body': {2}}}>".format(
+ self._timestamp, self._headers, self._body
+ )
+
+ def __init__(self, timestamp, headers, body):
+ self._timestamp = timestamp
+ self._headers = headers
+ self._body = body
diff --git a/tests/tools/sessionvalidation/session.py b/tests/tools/sessionvalidation/session.py
new file mode 100644
index 0000000..a36ae26
--- /dev/null
+++ b/tests/tools/sessionvalidation/session.py
@@ -0,0 +1,44 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sessionvalidation.transaction as transaction
+
+class Session(object):
+ ''' Session encapsulates a single user session '''
+
+ def getTransactionList(self):
+ ''' Returns a list of transaction objects '''
+ return self._transaction_list
+
+ def getTransactionIter(self):
+ ''' Returns an iterator of transaction objects '''
+ return iter(self._transaction_list)
+
+ def returnFirstTransaction(self):
+ return self._transaction_list[0]
+
+ def __repr__(self):
+ return "<Session {{'filename': {0}, 'version': {1}, 'timestamp: {2}, 'encoding': {3}, 'transaction_list': {4}}}>".format(
+ self._filename, self._version, self._timestamp, self._encoding, repr(self._transaction_list)
+ )
+
+ def __init__(self, filename, version, timestamp, transaction_list, encoding=None):
+ self._filename = filename
+ self._version = version
+ self._timestamp = timestamp
+ self._encoding = encoding
+ self._transaction_list = transaction_list
diff --git a/tests/tools/sessionvalidation/sessionvalidation.py b/tests/tools/sessionvalidation/sessionvalidation.py
new file mode 100644
index 0000000..c33b7c7
--- /dev/null
+++ b/tests/tools/sessionvalidation/sessionvalidation.py
@@ -0,0 +1,269 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import os
+
+import sessionvalidation.session as session
+import sessionvalidation.transaction as transaction
+import sessionvalidation.request as request
+import sessionvalidation.response as response
+
+valid_HTTP_request_methods = ['GET', 'POST', 'HEAD']
+G_VERBOSE_LOG = True
+
+
+def _verbose_print(msg, verbose_on=False):
+ ''' Print msg if verbose_on is set to True or G_VERBOSE_LOG is set to True'''
+ if verbose_on or G_VERBOSE_LOG:
+ print(msg)
+
+
+class SessionValidator(object):
+ '''
+ SessionValidator parses, validates, and exports an API for a given set of JSON sessions generated from Apache Traffic Server
+
+ SessionValidator is initialized with a path to a directory of JSON sessions. It then automatically parses and validates all the
+ session in the directory. After initialization, the user may use the provided API
+
+ TODO :
+ Provide a list of guaranteed fields for each type of object (ie a Transaction has a request and a response, a request has ...)
+ '''
+
+ def parse(self):
+ '''
+ Constructs Session objects from JSON files on disk and stores objects into _sessions
+
+ All sessions missing required fields (ie. a session timestamp, a response for every request, etc) are
+ dropped and the filename is stored inside _bad_sessions
+ '''
+
+ log_filenames = [os.path.join(self._json_log_dir, f) for f in os.listdir(self._json_log_dir) if os.path.isfile(os.path.join(self._json_log_dir, f))]
+
+ for fname in log_filenames:
+ with open(fname) as f:
+ # first attempt to load the JSON
+ try:
+ sesh = json.load(f)
+ except:
+ self._bad_sessions.append(fname)
+ _verbose_print("Warning: JSON parse error on file={0}".format(fname))
+ print("Warning: JSON parse error on file={0}".format(fname))
+ continue
+
+ # then attempt to extract all the required fields from the JSON
+ try:
+ session_timestamp = sesh['timestamp']
+ session_version = sesh['version']
+ session_txns = list()
+ for txn in sesh['txns']:
+ #print("PERSIA____________________________________________________________",txn)
+ # create transaction Request object
+ txn_request = txn['request']
+
+ txn_request_body = ''
+ if 'body' in txn_request:
+ txn_request_body = txn_request['body']
+ txn_request_obj = request.Request(txn_request['timestamp'], txn_request['headers'], txn_request_body)
+ # Create transaction Response object
+ txn_response = txn['response']
+ txn_response_body = ''
+ if 'body' in txn_response:
+ txn_response_body = txn_response['body']
+ txn_response_obj = response.Response(txn_response['timestamp'], txn_response['headers'], txn_response_body)
+
+ # create Transaction object
+ txn_obj = transaction.Transaction(txn_request_obj, txn_response_obj, txn['uuid'])
+ session_txns.append(txn_obj)
+ #print(txn_request['timestamp'])
+ session_obj = session.Session(fname, session_version, session_timestamp, session_txns)
+
+ except KeyError as e:
+ self._bad_sessions.append(fname)
+ print("Warning: parse error on key={0} for file={1}".format(e, fname))
+ _verbose_print("Warning: parse error on key={0} for file={1}".format(e, fname))
+ continue
+
+ self._sessions.append(session_obj)
+
+
+ def validate(self):
+ ''' Prunes out all the invalid Sessions in _sessions '''
+
+ good_sessions = list()
+
+ for sesh in self._sessions:
+ if SessionValidator.validateSingleSession(sesh):
+ good_sessions.append(sesh)
+ else:
+ self._bad_sessions.append(sesh._filename)
+
+ self._sessions = good_sessions
+
+
+ @staticmethod
+ def validateSingleSession(sesh):
+ ''' Takes in a single Session object as input, returns whether or not the Session is valid '''
+
+ retval = True
+
+ try:
+ # first validate fields
+ if not sesh._filename:
+ _verbose_print("bad session filename")
+ retval = False
+ elif not sesh._version:
+ _verbose_print("bad session version")
+ retval = False
+ elif float(sesh._timestamp) <= 0:
+ _verbose_print("bad session timestamp")
+ retval = False
+ elif not bool(sesh.getTransactionList()):
+ _verbose_print("session has no transaction list")
+ retval = False
+
+ # validate Transactions now
+ for txn in sesh.getTransactionIter():
+ if not SessionValidator.validateSingleTransaction(txn):
+ retval = False
+
+ except ValueError as e:
+ _verbose_print("most likely an invalid session timestamp")
+ retval = False
+
+ return retval
+
+
+ @staticmethod
+ def validateSingleTransaction(txn):
+ ''' Takes in a single Transaction object as input, and returns whether or not the Transaction is valid '''
+
+ txn_req = txn.getRequest()
+ txn_resp = txn.getResponse()
+ retval = True
+
+ #valid_HTTP_request_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH']
+ # we can later uncomment the previous line to support more HTTP methods
+ valid_HTTP_versions = ['HTTP/1.0', 'HTTP/1.1', 'HTTP/2.0']
+
+ try:
+ # validate request first
+ if not txn_req:
+ _verbose_print("no transaction request")
+ retval = False
+ elif txn_req.getBody() == None:
+ _verbose_print("transaction body is set to None")
+ retval = False
+ elif float(txn_req.getTimestamp()) <= 0:
+ _verbose_print("invalid transaction request timestamp")
+ retval = False
+ elif txn_req.getHeaders().split()[0] not in valid_HTTP_request_methods:
+ _verbose_print("invalid HTTP method for transaction {0}".format( txn_req.getHeaders().split()[0]))
+ retval = False
+ elif not txn_req.getHeaders().endswith("\r\n\r\n"):
+ _verbose_print("transaction request headers didn't end with \\r\\n\\r\\n")
+ retval = False
+ elif txn_req.getHeaders().split()[2] not in valid_HTTP_versions:
+ _verbose_print("invalid HTTP version in request")
+ retval = False
+
+ # if the Host header is not present and vaild we reject this transaction
+ found_host = False
+ for header in txn_req.getHeaders().split('\r\n'):
+ split_header = header.split(' ')
+ if split_header[0] == 'Host:':
+ found_host = True
+ host_header_no_space = len(split_header) == 1
+ host_header_with_space = len(split_header) == 2 and split_header[1] == ''
+ if host_header_no_space or host_header_with_space:
+ found_host = False
+ if not found_host:
+ print("missing host",txn_req)
+ _verbose_print("transaction request Host header doesn't have specified host")
+ retval = False
+
+ # reject if the host is localhost (since ATS seems to ignore remap rules for localhost requests)
+ if "127.0.0.1" in txn_req.getHeaders() or "localhost" in txn_req.getHeaders():
+ _verbose_print("transaction request Host is localhost, we must reject because ATS ignores remap rules for localhost requests")
+ retval = False
+
+
+ # now validate response
+ if not txn_resp:
+ _verbose_print("no transaction response")
+ retval = False
+ elif txn_resp.getBody() == None:
+ _verbose_print("transaction response body set to None")
+ retval = False
+ elif float(txn_resp.getTimestamp()) <= 0:
+ _verbose_print("invalid transaction response timestamp")
+ retval = False
+ elif txn_resp.getHeaders().split()[0] not in valid_HTTP_versions:
+ _verbose_print("invalid HTTP response header")
+ retval = False
+ elif not txn_resp.getHeaders().endswith("\r\n\r\n"):
+ _verbose_print("transaction response headers didn't end with \\r\\n\\r\\n")
+ retval = False
+
+ # if any of the 3xx responses have bodies, then the must reject this transaction, since 3xx
+ # errors by definition can't have bodies
+ response_line = txn_resp.getHeaders().split('\r\n')[0]
+ response_code = response_line.split(' ')[1]
+ if response_code.startswith('3') and txn_resp.getBody():
+ _verbose_print("transaction response was 3xx and had a body")
+ retval = False
+
+
+ except ValueError as e:
+ _verbose_print("most likely an invalid transaction timestamp")
+ retval = False
+
+ except IndexError as e:
+ _verbose_print("most likely a bad transaction header")
+ retval = False
+
+ return retval
+
+
+ def getSessionList(self):
+ ''' Returns the list of Session objects '''
+ return self._sessions
+
+
+ def getSessionIter(self):
+ ''' Returns an iterator of the Session objects '''
+ return iter(self._sessions)
+
+
+ def getBadSessionList(self):
+ ''' Returns a list of bad session filenames (list of strings) '''
+ return self._bad_sessions
+
+
+ def getBadSessionListIter(self):
+ ''' Returns an iterator of bad session filenames (iterator of strings) '''
+ return iter(self._bad_sessions)
+
+
+ def __init__(self, json_log_dir):
+ global valid_HTTP_request_methods
+ self._json_log_dir = json_log_dir
+ self._bad_sessions = list() # list of filenames
+ self._sessions = list() # list of _good_ session objects
+
+ self.parse()
+ self.validate()
diff --git a/tests/tools/sessionvalidation/transaction.py b/tests/tools/sessionvalidation/transaction.py
new file mode 100644
index 0000000..ca70de6
--- /dev/null
+++ b/tests/tools/sessionvalidation/transaction.py
@@ -0,0 +1,39 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sessionvalidation.request as request
+import sessionvalidation.response as response
+
+class Transaction(object):
+ ''' Tranaction encapsulates a single UA transaction '''
+
+ def getRequest(self):
+ return self._request
+
+ def getResponse(self):
+ return self._response
+
+ def __repr__(self):
+ return "<Transaction {{'uuid': {0}, 'request': {1}, 'response': {2}}}>".format(
+ self._uuid, self._request, self._response
+ )
+
+ def __init__(self, request, response, uuid):
+ self._request = request
+ self._response = response
+ self._uuid = uuid
diff --git a/tests/tools/traffic-replay/Config.py b/tests/tools/traffic-replay/Config.py
new file mode 100644
index 0000000..b4e2759
--- /dev/null
+++ b/tests/tools/traffic-replay/Config.py
@@ -0,0 +1,31 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#SSL config
+ca_certs = "/path/to/certfile"
+keyfile = "/path/to/keyfile"
+
+#Proxy config
+proxy_host = "127.0.0.1"
+proxy_ssl_port = 443
+proxy_nonssl_port = 8080
+
+#process and thread config
+nProcess = 4
+nThread = 4
diff --git a/tests/tools/traffic-replay/RandomReplay.py b/tests/tools/traffic-replay/RandomReplay.py
new file mode 100644
index 0000000..8c3de6f
--- /dev/null
+++ b/tests/tools/traffic-replay/RandomReplay.py
@@ -0,0 +1,167 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gevent
+import socket
+import requests
+import os
+from threading import Thread
+import sys
+from multiprocessing import current_process
+import sessionvalidation.sessionvalidation as sv
+import lib.result as result
+import extractHeader
+import mainProcess
+import json
+import gzip
+bSTOP = False
+def createDummyBodywithLength(numberOfbytes):
+ if numberOfbytes<=0:
+ return None
+ body= 'a'
+ while numberOfbytes!=1:
+ body += 'b'
+ numberOfbytes -= 1
+ return body
+
+def handleResponse(response,*args, **kwargs):
+ print(response.status_code)
+ #resp=args[0]
+ #expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2)
+ #expected_output = (int(expected_output_split[1]), str( expected_output_split[2]))
+ #r = result.Result(session_filename, expected_output[0], response.status_code)
+ #print(r.getResultString(colorize=True))
+# make sure len of the message body is greater than length
+def gen():
+ yield 'pforpersia,champaignurbana'.encode('utf-8')
+ yield 'there'.encode('utf-8')
+
+def txn_replay(session_filename, txn, proxy, result_queue, request_session):
+ """ Replays a single transaction
+ :param request_session: has to be a valid requests session"""
+ req = txn.getRequest()
+ resp = txn.getResponse()
+
+ # Construct HTTP request & fire it off
+ txn_req_headers = req.getHeaders()
+ txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers)
+ txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier
+ if 'body' in txn_req_headers_dict:
+ del txn_req_headers_dict['body']
+
+ #print("Replaying session")
+ try:
+ #response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers),
+ # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me
+ method = extractHeader.extract_txn_req_method(txn_req_headers)
+ response = None
+ body=None
+ content=None
+ if 'Transfer-Encoding' in txn_req_headers_dict:
+ # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header
+ # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this
+ # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh?
+ del txn_req_headers_dict['Host']
+ if 'Content-Length' in txn_req_headers_dict:
+ #print("ewww !")
+ del txn_req_headers_dict['Content-Length']
+ body = gen()
+ if 'Content-Length' in txn_req_headers_dict:
+ nBytes=int(txn_req_headers_dict['Content-Length'])
+ body = createDummyBodywithLength(nBytes)
+ #print("request session is",id(request_session))
+ if method == 'GET':
+ response = request_session.get('http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ headers=txn_req_headers_dict, stream=False, allow_redirects=False,data=body)
+ if 'Content-Length' in response.headers:
+ content = response.raw
+ #print("len: {0} received {1}".format(response.headers['Content-Length'],content))
+
+ elif method == 'POST':
+ response = request_session.post('http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ headers=txn_req_headers_dict, stream=False, data=body, allow_redirects=False)
+
+ if 'Content-Length' in response.headers:
+ content = response.raw
+ #print("reading==========>>>>>>>>>>>>>.")
+ #print(content.data)
+ #print("len: {0} received {1}".format(response.headers['Content-Length'],content))
+ elif method == 'HEAD':
+ response = request_session.head('http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ headers=txn_req_headers_dict, stream=False)
+
+ #gzip_file = gzip.GzipFile(fileobj=content)
+ #shutil.copyfileobj(gzip_file, f)
+ expected=extractHeader.responseHeader_to_dict(resp.getHeaders())
+ #print(expected)
+ if mainProcess.verbose:
+ expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2)
+ expected_output = (int(expected_output_split[1]), str( expected_output_split[2]))
+ r = result.Result(session_filename, expected_output[0], response.status_code)
+ print(r.getResultString(response.headers,expected,colorize=True))
+ r.Compare(response.headers,expected)
+ #result_queue.put(r)
+ except UnicodeEncodeError as e:
+ # these unicode errors are due to the interaction between Requests and our wiretrace data.
+ # TODO fix
+ print("UnicodeEncodeError exception")
+
+ except requests.exceptions.ContentDecodingError as e:
+ print("ContentDecodingError",e)
+ except:
+ e=sys.exc_info()
+ print("ERROR in requests: ",e,response, session_filename)
+
+def session_replay(input, proxy, result_queue):
+ global bSTOP
+ ''' Replay all transactions in session
+
+ This entire session will be replayed in one requests.Session (so one socket / TCP connection)'''
+ #if timing_control:
+ # time.sleep(float(session._timestamp)) # allow other threads to run
+ while bSTOP == False:
+ for session in iter(input.get, 'STOP'):
+ #print(bSTOP)
+ if session == 'STOP':
+ print("Queue is empty")
+ bSTOP = True
+ break
+ with requests.Session() as request_session:
+ request_session.proxies = proxy
+ for txn in session.getTransactionIter():
+ try:
+ txn_replay(session._filename, txn, proxy, result_queue, request_session)
+ except:
+ e=sys.exc_info()
+ print("ERROR in replaying: ",e,txn.getRequest().getHeaders())
+ bSTOP = True
+ print("Queue is empty")
+ input.put('STOP')
+ break
+
+def client_replay(input, proxy, result_queue, nThread):
+ Threads = []
+ for i in range(nThread):
+ t = Thread(target=session_replay, args=[input, proxy, result_queue])
+ t.start()
+ Threads.append(t)
+
+ for t1 in Threads:
+ t1.join()
diff --git a/tests/tools/traffic-replay/SSLReplay.py b/tests/tools/traffic-replay/SSLReplay.py
new file mode 100644
index 0000000..dfc7d5c
--- /dev/null
+++ b/tests/tools/traffic-replay/SSLReplay.py
@@ -0,0 +1,213 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import socket, ssl, pprint
+import gevent
+import requests
+import os
+#import threading
+import sys
+from multiprocessing import current_process
+import sessionvalidation.sessionvalidation as sv
+import lib.result as result
+import extractHeader
+from gevent import monkey, sleep
+from threading import Thread
+import mainProcess
+import json
+import extractHeader
+import time
+import Config
+bSTOP = False
+
+class ProxyHTTPSConnection(http.client.HTTPSConnection):
+ "This class allows communication via SSL."
+
+ default_port = http.client.HTTPS_PORT
+
+ # XXX Should key_file and cert_file be deprecated in favour of context?
+
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None, *, context=None,
+ check_hostname=None,server_name = None):
+ #http.client.HTTPSConnection.__init__(self)
+ super().__init__(host, port, key_file,cert_file, timeout, source_address,context=context,check_hostname=check_hostname)
+ '''
+ self.key_file = key_file
+ self.cert_file = cert_file
+ if context is None:
+ context = ssl._create_default_https_context()
+ will_verify = context.verify_mode != ssl.CERT_NONE
+ if check_hostname is None:
+ check_hostname = context.check_hostname
+ if check_hostname and not will_verify:
+ raise ValueError("check_hostname needs a SSL context with "
+ "either CERT_OPTIONAL or CERT_REQUIRED")
+ if key_file or cert_file:
+ context.load_cert_chain(cert_file, key_file)
+ self._context = context
+ self._check_hostname = check_hostname
+ '''
+ self.server_name = server_name
+
+ def connect(self):
+ "Connect to a host on a given (SSL) port."
+ http.client.HTTPConnection.connect(self)
+
+ if self._tunnel_host:
+ server_hostname = self._tunnel_host
+ else:
+ server_hostname = self.server_name
+ self.sock = self._context.wrap_socket(self.sock,
+ do_handshake_on_connect=True,
+ server_side=False,
+ server_hostname=server_hostname)
+ if not self._context.check_hostname and self._check_hostname:
+ try:
+ ssl.match_hostname(self.sock.getpeercert(), server_hostname)
+ except Exception:
+ self.sock.shutdown(socket.SHUT_RDWR)
+ self.sock.close()
+ raise
+
+
+def txn_replay(session_filename, txn, proxy, result_queue, request_session):
+ """ Replays a single transaction
+ :param request_session: has to be a valid requests session"""
+ req = txn.getRequest()
+ resp = txn.getResponse()
+ responseDict = {}
+ # Construct HTTP request & fire it off
+ txn_req_headers = req.getHeaders()
+ txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers)
+ txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier
+ if 'body' in txn_req_headers_dict:
+ del txn_req_headers_dict['body']
+
+ #print("Replaying session")
+ try:
+ #response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers),
+ # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me
+ method = extractHeader.extract_txn_req_method(txn_req_headers)
+ response = None
+ body=None
+ content=None
+ if 'Transfer-Encoding' in txn_req_headers_dict:
+ # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header
+ # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this
+ # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh?
+ del txn_req_headers_dict['Host']
+ if 'Content-Length' in txn_req_headers_dict:
+ #print("ewww !")
+ del txn_req_headers_dict['Content-Length']
+ body = gen()
+ if 'Content-Length' in txn_req_headers_dict:
+ nBytes=int(txn_req_headers_dict['Content-Length'])
+ body = createDummyBodywithLength(nBytes)
+ #print("request session is",id(request_session))
+ if method == 'GET':
+ request_session.request('GET','https://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ headers=txn_req_headers_dict,body=body)
+ r1 = request_session.getresponse()
+ responseHeaders = r1.getheaders()
+ responseContent = r1.read()
+
+ elif method == 'POST':
+ request_session.request('POST','https://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ headers=txn_req_headers_dict,body=body)
+ r1 = request_session.getresponse()
+ responseHeaders = r1.getheaders()
+ responseContent = r1.read()
+
+ elif method == 'HEAD':
+ request_session.request('HEAD','https://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ headers=txn_req_headers_dict,body=body)
+ r1 = request_session.getresponse()
+ responseHeaders = r1.getheaders()
+ responseContent = r1.read()
+ for key,value in responseHeaders:
+ responseDict[key.lower()] = value
+ expected=extractHeader.responseHeader_to_dict(resp.getHeaders())
+ #print(responseDict)
+ if mainProcess.verbose:
+ expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2)
+ expected_output = (int(expected_output_split[1]), str( expected_output_split[2]))
+ r = result.Result(session_filename, expected_output[0],r1.status)
+ print(r.getResultString(responseDict,expected,colorize=True))
+ r.Compare(responseDict,expected)
+ #result_queue.put(r)
+ except UnicodeEncodeError as e:
+ # these unicode errors are due to the interaction between Requests and our wiretrace data.
+ # TODO fix
+ print("UnicodeEncodeError exception")
+
+ except requests.exceptions.ContentDecodingError as e:
+ print("ContentDecodingError",e)
+ except:
+ e=sys.exc_info()
+ print("ERROR in requests: ",e,response, session_filename)
+def client_replay(input, proxy, result_queue, nThread):
+ Threads = []
+ for i in range(nThread):
+ t = Thread(target=session_replay, args=[input, proxy, result_queue])
+ t.start()
+ Threads.append(t)
+
+ for t1 in Threads:
+ t1.join()
+
+def session_replay(input, proxy, result_queue):
+
+ ''' Replay all transactions in session
+
+ This entire session will be replayed in one requests.Session (so one socket / TCP connection)'''
+ #if timing_control:
+ # time.sleep(float(session._timestamp)) # allow other threads to run
+ global bSTOP
+ sslSocks = []
+ while bSTOP == False:
+ for session in iter(input.get, 'STOP'):
+ txn = session.returnFirstTransaction()
+ req = txn.getRequest()
+ # Construct HTTP request & fire it off
+ txn_req_headers = req.getHeaders()
+ txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers)
+ sc = ssl.SSLContext(protocol=ssl.PROTOCOL_SSLv23)
+ sc.load_cert_chain(Config.ca_certs,keyfile=Config.keyfile)
+ conn = ProxyHTTPSConnection(Config.proxy_host,Config.proxy_ssl_port,cert_file=Config.ca_certs,key_file=Config.keyfile,context=sc,server_name=txn_req_headers_dict['Host'])
+ for txn in session.getTransactionIter():
+ try:
+ #print(txn._uuid)
+ txn_replay(session._filename, txn, proxy, result_queue, conn)
+ except:
+ e=sys.exc_info()
+ print("ERROR in replaying: ",e,txn.getRequest().getHeaders())
+ #sslSocket.bStop = False
+
+ bSTOP = True
+ print("stopping now")
+ input.put('STOP')
+ break
+
+ #time.sleep(0.5)
+ for sslSock in sslSocks:
+ sslSock.ssl_sock.close()
diff --git a/tests/tools/traffic-replay/Scheduler.py b/tests/tools/traffic-replay/Scheduler.py
new file mode 100644
index 0000000..8d2d6d0
--- /dev/null
+++ b/tests/tools/traffic-replay/Scheduler.py
@@ -0,0 +1,62 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import random
+import json
+from multiprocessing import Process, Queue, current_process
+from progress.bar import Bar
+import sessionvalidation.sessionvalidation as sv
+import WorkerTask
+import time
+
+
+def LaunchWorkers(path,nProcess,proxy,replay_type, nThread):
+ ms1=time.time()
+ s = sv.SessionValidator(path)
+ sessions = s.getSessionList()
+ sessions.sort(key=lambda session: session._timestamp)
+ Processes=[]
+ Qsize = 25000 #int (1.5 * len(sessions)/(nProcess))
+ QList=[Queue(Qsize) for i in range(nProcess)]
+ print("Dropped {0} sessions for being malformed. Number of correct sessions {1}".format(len(s.getBadSessionList()),len(sessions)))
+ print(range(nProcess))
+ OutputQ=Queue();
+ #======================================== Pre-load queues
+ for session in sessions:
+ #if nProcess == 1:
+ # QList[0].put(session)
+ #else:
+ QList[random.randint(0,nProcess-1)].put(session)
+ #if QList[0].qsize() > 10 :
+ # break
+ #=============================================== Launch Processes
+ print("size",QList[0].qsize())
+ for i in range(nProcess):
+ QList[i].put('STOP')
+ for i in range(nProcess):
+ p=Process(target=WorkerTask.worker, args=[QList[i],OutputQ,proxy,replay_type, nThread])
+ p.daemon=False
+ Processes.append(p);
+ p.start()
+
+ for p in Processes:
+ p.join()
+ ms2=time.time()
+ print("OK enough, it is time to exit, running time in seconds", (ms2-ms1))
diff --git a/tests/tools/traffic-replay/WorkerTask.py b/tests/tools/traffic-replay/WorkerTask.py
new file mode 100644
index 0000000..e5ba97d
--- /dev/null
+++ b/tests/tools/traffic-replay/WorkerTask.py
@@ -0,0 +1,45 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+import requests
+import os
+#import threading
+import sys
+from multiprocessing import current_process
+import sessionvalidation.sessionvalidation as sv
+import lib.result as result
+from progress.bar import Bar
+import extractHeader
+import RandomReplay
+import SSLReplay
+import h2Replay
+def worker(input,output,proxy,replay_type,nThread):
+ #progress_bar = Bar(" Replaying sessions {0}".format(current_process().name), max=input.qsize())
+ #print("playing {0}=>{1}:{2}".format(current_process().name,session._timestamp,proxy))
+ if replay_type == 'random':
+ RandomReplay.client_replay(input, proxy, output, nThread)
+ elif replay_type == 'ssl':
+ SSLReplay.client_replay(input, proxy, output,nThread)
+ elif replay_type == 'h2':
+ h2Replay.client_replay(input, proxy, output,nThread)
+ #progress_bar.next()
+ #progress_bar.finish()
+ print("process{0} has exited".format(current_process().name))
+
diff --git a/tests/tools/traffic-replay/__main__.py b/tests/tools/traffic-replay/__main__.py
new file mode 100644
index 0000000..7f2fcc5
--- /dev/null
+++ b/tests/tools/traffic-replay/__main__.py
@@ -0,0 +1,35 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+import mainProcess
+import argparse
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-type",action='store', dest='replay_type', help="Replay type: ssl/random/h2")
+ parser.add_argument("-log_dir",type=str, help="directory of JSON replay files")
+ parser.add_argument("-v", dest="verbose", help="verify response status code", action="store_true")
+
+ args = parser.parse_args()
+
+ # Let 'er loose
+ #main(args.log_dir, args.hostname, int(args.port), args.threads, args.timing, args.verbose)
+ mainProcess.main(args.log_dir, args.replay_type,args.verbose)
diff --git a/tests/tools/traffic-replay/extractHeader.py b/tests/tools/traffic-replay/extractHeader.py
new file mode 100644
index 0000000..5a52f05
--- /dev/null
+++ b/tests/tools/traffic-replay/extractHeader.py
@@ -0,0 +1,64 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sessionvalidation
+def extract_txn_req_method(headers):
+ ''' Extracts the HTTP request method from the header in a string format '''
+ line = (headers.split('\r\n'))[0]
+ return (line.split(' '))[0]
+
+def extract_host(headers):
+ ''' Returns the host header from the given headers '''
+ lines = headers.split('\r\n')
+ for line in lines:
+ if 'Host:' in line:
+ return line.split(' ')[1]
+ return "notfound"
+
+def responseHeader_to_dict(header):
+ headerFields = header.split('\r\n',1)[1]
+ fields =headerFields.split('\r\n')
+ header = [x for x in fields if (x != u'')]
+ headers = {}
+ for line in header:
+ split_here = line.find(":")
+ headers[line[:split_here].lower()] = line[(split_here + 1):].strip()
+
+ return headers
+
+def header_to_dict(header):
+ ''' Convert a HTTP header in string format to a python dictionary
+ Returns a dictionary of header values
+ '''
+ header = header.split('\r\n')
+ header = [x for x in header if (x != u'')]
+ headers = {}
+ for line in header:
+ if 'GET' in line or 'POST' in line or 'HEAD' in line: # ignore initial request line
+ continue
+
+ split_here = line.find(":")
+ headers[line[:split_here]] = line[(split_here + 1):].strip()
+
+ return headers
+
+def extract_GET_path(headers):
+ ''' Extracts the HTTP request URL from the header in a string format '''
+ line = (headers.split('\r\n'))[0]
+ return (line.split(' '))[1]
diff --git a/tests/tools/traffic-replay/h2Replay.py b/tests/tools/traffic-replay/h2Replay.py
new file mode 100644
index 0000000..31224bc
--- /dev/null
+++ b/tests/tools/traffic-replay/h2Replay.py
@@ -0,0 +1,327 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gevent
+import os
+from threading import Thread
+import sys
+from multiprocessing import current_process
+import sessionvalidation.sessionvalidation as sv
+import lib.result as result
+import extractHeader
+import mainProcess
+import json
+from hyper import HTTP20Connection
+from hyper.tls import wrap_socket, H2_NPN_PROTOCOLS, H2C_PROTOCOL
+from hyper.common.bufsocket import BufferedSocket
+import hyper
+import socket
+import logging
+import h2
+from h2.connection import H2Configuration
+import threading
+import Config
+
+log = logging.getLogger(__name__)
+bSTOP = False
+hyper.tls._context = hyper.tls.init_context()
+hyper.tls._context.check_hostname = False
+hyper.tls._context.verify_mode = hyper.compat.ssl.CERT_NONE
+class _LockedObject(object):
+ """
+ A wrapper class that hides a specific object behind a lock.
+
+ The goal here is to provide a simple way to protect access to an object
+ that cannot safely be simultaneously accessed from multiple threads. The
+ intended use of this class is simple: take hold of it with a context
+ manager, which returns the protected object.
+ """
+ def __init__(self, obj):
+ self.lock = threading.RLock()
+ self._obj = obj
+
+ def __enter__(self):
+ self.lock.acquire()
+ return self._obj
+
+ def __exit__(self, _exc_type, _exc_val, _exc_tb):
+ self.lock.release()
+
+
+class h2ATS(HTTP20Connection):
+
+
+ def __init_state(self):
+ """
+ Initializes the 'mutable state' portions of the HTTP/2 connection
+ object.
+
+ This method exists to enable HTTP20Connection objects to be reused if
+ they're closed, by resetting the connection object to its basic state
+ whenever it ends up closed. Any situation that needs to recreate the
+ connection can call this method and it will be done.
+
+ This is one of the only methods in hyper that is truly private, as
+ users should be strongly discouraged from messing about with connection
+ objects themselves.
+ """
+
+ config1 = H2Configuration(
+ client_side=True,
+ header_encoding='utf-8',
+ validate_outbound_headers=False,
+ validate_inbound_headers=False,
+
+ )
+ self._conn = _LockedObject(h2.connection.H2Connection(config=config1))
+
+ # Streams are stored in a dictionary keyed off their stream IDs. We
+ # also save the most recent one for easy access without having to walk
+ # the dictionary.
+ #
+ # We add a set of all streams that we or the remote party forcefully
+ # closed with RST_STREAM, to avoid encountering issues where frames
+ # were already in flight before the RST was processed.
+ #
+ # Finally, we add a set of streams that recently received data. When
+ # using multiple threads, this avoids reading on threads that have just
+ # acquired the I/O lock whose streams have already had their data read
+ # for them by prior threads.
+ self.streams = {}
+ self.recent_stream = None
+ self.next_stream_id = 1
+ self.reset_streams = set()
+ self.recent_recv_streams = set()
+
+ # The socket used to send data.
+ self._sock = None
+
+ # Instantiate a window manager.
+ #self.window_manager = self.__wm_class(65535)
+
+ return
+
+ def __init__(self,host,**kwargs):
+ HTTP20Connection.__init__(self,host,**kwargs)
+ self.__init_state()
+
+ def connect(self):
+ """
+ Connect to the server specified when the object was created. This is a
+ no-op if we're already connected.
+
+ Concurrency
+ -----------
+
+ This method is thread-safe. It may be called from multiple threads, and
+ is a noop for all threads apart from the first.
+
+ :returns: Nothing.
+
+ """
+ #print("connecting to ATS")
+ with self._lock:
+ if self._sock is not None:
+ return
+ sni = self.host
+ if not self.proxy_host:
+ host = self.host
+ port = self.port
+ else:
+ host = self.proxy_host
+ port = self.proxy_port
+
+ sock = socket.create_connection((host, port))
+
+ if self.secure:
+ #assert not self.proxy_host, "Proxy with HTTPS not supported."
+ sock, proto = wrap_socket(sock, sni, self.ssl_context,
+ force_proto=self.force_proto)
+ else:
+ proto = H2C_PROTOCOL
+
+ log.debug("Selected NPN protocol: %s", proto)
+ assert proto in H2_NPN_PROTOCOLS or proto == H2C_PROTOCOL
+
+ self._sock = BufferedSocket(sock, self.network_buffer_size)
+
+ self._send_preamble()
+
+def createDummyBodywithLength(numberOfbytes):
+ if numberOfbytes==0:
+ return None
+ body= 'a'
+ while numberOfbytes!=1:
+ body += 'b'
+ numberOfbytes -= 1
+ return body
+
+def handleResponse(response,*args, **kwargs):
+ print(response.status_code)
+ #resp=args[0]
+ #expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2)
+ #expected_output = (int(expected_output_split[1]), str( expected_output_split[2]))
+ #r = result.Result(session_filename, expected_output[0], response.status_code)
+ #print(r.getResultString(colorize=True))
+# make sure len of the message body is greater than length
+def gen():
+ yield 'pforpersia,champaignurbana'.encode('utf-8')
+ yield 'there'.encode('utf-8')
+
+def txn_replay(session_filename, txn, proxy, result_queue, h2conn,request_IDs):
+ """ Replays a single transaction
+ :param request_session: has to be a valid requests session"""
+ req = txn.getRequest()
+ resp = txn.getResponse()
+ # Construct HTTP request & fire it off
+ txn_req_headers = req.getHeaders()
+ txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers)
+ txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier
+ if 'body' in txn_req_headers_dict:
+ del txn_req_headers_dict['body']
+ responseID = -1
+ #print("Replaying session")
+ try:
+ #response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers),
+ # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers),
+ # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me
+ method = extractHeader.extract_txn_req_method(txn_req_headers)
+ response = None
+ mbody=None
+ #txn_req_headers_dict['Host'] = "localhost"
+ if 'Transfer-Encoding' in txn_req_headers_dict:
+ # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header
+ # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this
+ # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh?
+ #del txn_req_headers_dict['Host']
+ if 'Content-Length' in txn_req_headers_dict:
+ #print("ewww !")
+ del txn_req_headers_dict['Content-Length']
+ mbody = gen()
+ if 'Content-Length' in txn_req_headers_dict:
+ nBytes=int(txn_req_headers_dict['Content-Length'])
+ mbody = createDummyBodywithLength(nBytes)
+ if 'Connection' in txn_req_headers_dict:
+ del txn_req_headers_dict['Connection']
+ #str2 = extractHeader.extract_host(txn_req_headers)+ extractHeader.extract_GET_path(txn_req_headers)
+ #print(str2)
+ if method == 'GET':
+ responseID = h2conn.request('GET',url=extractHeader.extract_GET_path(txn_req_headers),headers=txn_req_headers_dict, body=mbody)
+ #print("get response", responseID)
+ return responseID
+ #request_IDs.append(responseID)
+ #response = h2conn.get_response(id)
+ #print(response.headers)
+ #if 'Content-Length' in response.headers:
+ # content = response.read()
+ #print("len: {0} received {1}".format(response.headers['Content-Length'],content))
+
+ elif method == 'POST':
+ responseID = h2conn.request('POST',url=extractHeader.extract_GET_path(txn_req_headers),headers=txn_req_headers_dict, body=mbody)
+ print("get response", responseID)
+ return responseID
+
+ elif method == 'HEAD':
+ responseID = h2conn.request('HEAD',url=extractHeader.extract_GET_path(txn_req_headers),headers=txn_req_headers_dict)
+ print("get response", responseID)
+ return responseID
+
+ #print(response.headers)
+ #print("logged respose")
+ expected=extractHeader.responseHeader_to_dict(resp.getHeaders())
+ #print(expected)
+ if mainProcess.verbose:
+ expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2)
+ expected_output = (int(expected_output_split[1]), str( expected_output_split[2]))
+ r = result.Result(session_filename, expected_output[0], response.status_code)
+ print(r.getResultString(response.headers,expected,colorize=True))
+
+ #return responseID
+
+ except UnicodeEncodeError as e:
+ # these unicode errors are due to the interaction between Requests and our wiretrace data.
+ # TODO fix
+ print("UnicodeEncodeError exception")
+
+ except:
+ e=sys.exc_info()
+ print("ERROR in requests: ",e,response, session_filename)
+
+def session_replay(input, proxy, result_queue):
+ global bSTOP
+ ''' Replay all transactions in session
+
+ This entire session will be replayed in one requests.Session (so one socket / TCP connection)'''
+ #if timing_control:
+ # time.sleep(float(session._timestamp)) # allow other threads to run
+ while bSTOP == False:
+ for session in iter(input.get, 'STOP'):
+ print(bSTOP)
+ if session == 'STOP':
+ print("Queue is empty")
+ bSTOP = True
+ break
+ txn = session.returnFirstTransaction()
+ req = txn.getRequest()
+ # Construct HTTP request & fire it off
+ txn_req_headers = req.getHeaders()
+ txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers)
+ with h2ATS(txn_req_headers_dict['Host'], secure=True, proxy_host=Config.proxy_host,proxy_port=Config.proxy_ssl_port) as h2conn:
+ request_IDs = []
+ respList = []
+ for txn in session.getTransactionIter():
+ try:
+ ret = txn_replay(session._filename, txn, proxy, result_queue, h2conn,request_IDs)
+ respList.append(txn.getResponse())
+ request_IDs.append(ret)
+ #print("txn return value is ",ret)
+ except:
+ e=sys.exc_info()
+ print("ERROR in replaying: ",e,txn.getRequest().getHeaders())
+ for id in request_IDs:
+ expectedH = respList.pop(0);
+ #print("extracting",id)
+ response = h2conn.get_response(id)
+ #print("code {0}:{1}".format(response.status,response.headers))
+ response_dict = {}
+ if mainProcess.verbose:
+ for field,value in response.headers.items():
+ response_dict[field.decode('utf-8')] = value.decode('utf-8')
+
+ expected_output_split = expectedH.getHeaders().split('\r\n')[ 0].split(' ', 2)
+ expected_output = (int(expected_output_split[1]), str( expected_output_split[2]))
+ r = result.Result("", expected_output[0], response.status)
+ expected_Dict = extractHeader.responseHeader_to_dict(expectedH.getHeaders())
+ print(r.getResultString(response_dict,expected_Dict,colorize=True))
+ #r.Compare(response_dict,expected_Dict)
+
+ bSTOP = True
+ print("Queue is empty")
+ input.put('STOP')
+ break
+
+def client_replay(input, proxy, result_queue, nThread):
+ Threads = []
+ for i in range(nThread):
+ t = Thread(target=session_replay, args=[input, proxy, result_queue])
+ t.start()
+ Threads.append(t)
+
+ for t1 in Threads:
+ t1.join()
diff --git a/tests/tools/traffic-replay/mainProcess.py b/tests/tools/traffic-replay/mainProcess.py
new file mode 100644
index 0000000..a2f38f0
--- /dev/null
+++ b/tests/tools/traffic-replay/mainProcess.py
@@ -0,0 +1,63 @@
+#!/bin/env python3
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+import json
+import socket
+import os
+import threading
+import time
+import argparse
+import subprocess
+import shlex
+from multiprocessing import Pool, Process
+from collections import deque
+from progress.bar import Bar
+import sessionvalidation.sessionvalidation as sv
+import lib.result as result
+import WorkerTask
+import Scheduler
+import Config
+verbose = False
+def check_for_ats(hostname, port):
+ ''' Checks to see if ATS is running on `hostname` and `port`
+ If not running, this function will terminate the script
+ '''
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ result = sock.connect_ex((hostname, port))
+ if result != 0:
+ # hostname:port is not being listened to
+ print('==========')
+ print('Error: Apache Traffic Server is not running on {0}:{1}'.format(hostname, port))
+ print('Aborting')
+ print('==========')
+ sys.exit()
+# Note: this function can't handle multi-line (ie wrapped line) headers
+# Hopefully this isn't an issue because multi-line headers are deprecated now
+
+def main(path, replay_type, Bverbose):
+ global verbose
+ verbose = Bverbose
+ check_for_ats(Config.proxy_host, Config.proxy_nonssl_port)
+ proxy = {"http": "http://{0}:{1}".format(Config.proxy_host, Config.proxy_nonssl_port)}
+ Scheduler.LaunchWorkers(path,Config.nProcess,proxy,replay_type, Config.nThread)
+
+
+
--
To stop receiving notification emails like this one, please contact
"commits@trafficserver.apache.org" <co...@trafficserver.apache.org>.