You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by am...@apache.org on 2013/08/10 02:38:47 UTC
[1/2] Documentation: Added 'arch' (Architecture) section. Added cache
documents. Fixed many warnings. Added Tiered Storage proposal.
Updated Branches:
refs/heads/master 8ecd7e66c -> 7c21e68ef
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/ext/traffic-server.py
----------------------------------------------------------------------
diff --git a/doc/ext/traffic-server.py b/doc/ext/traffic-server.py
index 2a978f2..36959fb 100644
--- a/doc/ext/traffic-server.py
+++ b/doc/ext/traffic-server.py
@@ -149,12 +149,42 @@ class TrafficServerDomain(Domain):
for var, doc in self.data['cv'].iteritems():
yield var, var, 'cv', doc, var, 1
+# These types are ignored as missing references for the C++ domain.
+EXTERNAL_TYPES = set((
+ 'int',
+ 'uint8_t', 'uint16_t', 'uint24_t', 'uint32_t', 'uint64_t',
+ 'off_t',
+ 'Event', 'INK_MD5', 'DLL<EvacuationBlock>',
+ ))
+
+# Clean up specific references that we know will never be defined but are implicitly used by
+# other domain directives. Hand convert them to literals.
+def xref_cleanup(app, env, node, contnode):
+ rdomain = node['refdomain']
+ rtype = node['reftype']
+ rtarget = node['reftarget']
+ if 'cpp' == rdomain:
+ if 'type' == rtype:
+ if rtarget in EXTERNAL_TYPES:
+ node = nodes.literal()
+ node += contnode
+ return node
+ # allow pointer or reference decorations
+ if ('*' == rtarget[-1] or '&' == rtarget[-1]) and rtarget[:-1] in EXTERNAL_TYPES:
+ return contnode
+ return;
+
def setup(app):
app.add_crossref_type('configfile', 'file',
objname='Configuration file',
indextemplate='pair: %s; Configuration files')
+
+ rst.roles.register_generic_role('arg', nodes.emphasis);
+
app.add_domain(TrafficServerDomain)
# Types that we want the C domain to consider built in
- for word in ('size_t'):
+ for word in EXTERNAL_TYPES:
sphinx.domains.c.CObject.stopwords.add(word)
+
+ app.connect('missing-reference', xref_cleanup)
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/index.rst
----------------------------------------------------------------------
diff --git a/doc/index.rst b/doc/index.rst
index 951bd86..a665ea1 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -43,6 +43,7 @@ bandwidth.
reference/commands/index.en
reference/plugins/index.en
reference/api/index.en
+ arch/index.en
Indices and tables
==================
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/reference/api/TSHttpHookAdd.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/api/TSHttpHookAdd.en.rst b/doc/reference/api/TSHttpHookAdd.en.rst
index 657b336..0200fc7 100644
--- a/doc/reference/api/TSHttpHookAdd.en.rst
+++ b/doc/reference/api/TSHttpHookAdd.en.rst
@@ -28,6 +28,13 @@ Synopsis
.. function:: void TSHttpSsnHookAdd(TSHttpSsn ssnp, TSHttpHookID id, TSCont contp)
.. function:: void TSHttpTxnHookAdd(TSHttpTxn txnp, TSHttpHookID id, TSCont contp)
+.. rubric:: Opaque types
+
+.. type:: TSHttpTxn
+.. type:: TSHttpSsn
+.. type:: TSHttpHookID
+.. type:: TSCont
+
Description
===========
@@ -46,20 +53,20 @@ transaction. :func:`TSHttpHookAdd` is typically called from
A session consists of a single client connection to Traffic Server.
A session can consist of several transactions in succession. The
session starts when the client connection opens, and ends when the
-connection closes. :func:`TSHttpSsnHookAdd` adds :data:`contp` to
-the end of the list of HTTP transaction hooks specified by :data:`id`.
-This means that :data:`contp` is called back for every transaction
+connection closes. :func:`TSHttpSsnHookAdd` adds :arg:`contp` to
+the end of the list of HTTP transaction hooks specified by :arg:`id`.
+This means that :arg:`contp` is called back for every transaction
within the session, at the point specified by the hook ID. Since
-:data:`contp` is added to a session, it is not possible to call
+:arg:`contp` is added to a session, it is not possible to call
:func:`TSHttpSsnHookAdd` from the plugin initialization routine;
the plugin needs a handle to an HTTP session.
A transaction consists of a single HTTP request from a client and
the response that Traffic Server sends to that client. A transaction
begins when Traffic Server receives a request, and ends when Traffic
-Server sends the response. :func:`TSHttpTxnHookAdd` adds :data:`contp`
+Server sends the response. :func:`TSHttpTxnHookAdd` adds :arg:`contp`
to the end of the list of HTTP transaction hooks specified by
-:data:`id`. Since :data:`contp` is added to a transaction, it is
+:arg:`id`. Since :arg:`contp` is added to a transaction, it is
not possible to call :func:`TSHttpTxnHookAdd` from the plugin
initialization routine but only when the plugin has a handle to an
HTTP transaction.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/reference/configuration/ip_allow.config.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/configuration/ip_allow.config.en.rst b/doc/reference/configuration/ip_allow.config.en.rst
index 49812a1..f8cc67b 100644
--- a/doc/reference/configuration/ip_allow.config.en.rst
+++ b/doc/reference/configuration/ip_allow.config.en.rst
@@ -1,19 +1,19 @@
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
===============
ip_allow.config
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/reference/configuration/records.config.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/configuration/records.config.en.rst b/doc/reference/configuration/records.config.en.rst
index 1ca3cfb..6ff82fa 100644
--- a/doc/reference/configuration/records.config.en.rst
+++ b/doc/reference/configuration/records.config.en.rst
@@ -1064,6 +1064,10 @@ Customizable User Response Pages
- ``1`` = always suppress generated response pages
- ``2`` = suppress response pages only for intercepted traffic
+.. ts:cv:: CONFIG proxy.config.http_ui_enabled INT 0
+
+ Enable the user interface page.
+
DNS
===
@@ -1090,9 +1094,11 @@ hostname to ``host_x.y.com``.
Traffic Server to add the hostname extension .org, then specify ``org`` as the value for this variable (Traffic Server automatically
adds the dot (.)).
- **Note:** If the variable
- `proxy.config.http.enable_url_expandomatic`_ is set to ``1`` (the default value), then you do not have to add *``www.``* and
- *``.com``* to this list because Traffic Server automatically tries www. and .com after trying the values you've specified.
+.. note::
+
+ If the variable :ts:cv:`proxy.config.http.enable_url_expandomatic` is set to ``1`` (the default value), then you do not have to
+ add *``www.``* and *``.com``* to this list because Traffic Server automatically tries www. and .com after trying the values
+ you've specified.
.. ts:cv:: CONFIG proxy.config.dns.resolv_conf STRING /etc/resolv.conf
@@ -1554,6 +1560,10 @@ SSL Termination
:file:`ssl_multicert.config` will be loaded relative to this
path.
+.. ts:cv:: CONFIG proxy.config.ssl.server.private_key.filename STRING NULL
+
+ TBD
+
.. ts:cv:: CONFIG proxy.config.ssl.server.cert_chain.filename STRING NULL
The name of a file containing a global certificate chain that
@@ -1572,6 +1582,10 @@ SSL Termination
The filename of the certificate authority that client certificates
will be verified against.
+.. ts:cv:: CONFIG proxy.config.ssl.auth.enabled INT 0
+
+ TBD
+
Client-Related Configuration
----------------------------
[2/2] git commit: Documentation: Added 'arch' (Architecture) section.
Added cache documents. Fixed many warnings. Added Tiered Storage proposal.
Posted by am...@apache.org.
Documentation:
Added 'arch' (Architecture) section.
Added cache documents.
Fixed many warnings.
Added Tiered Storage proposal.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/7c21e68e
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/7c21e68e
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/7c21e68e
Branch: refs/heads/master
Commit: 7c21e68ef1aca3d4c1a372d50638742777f6de43
Parents: 8ecd7e6
Author: Alan M. Carroll <am...@network-geographics.com>
Authored: Fri Aug 9 19:37:52 2013 -0500
Committer: Alan M. Carroll <am...@network-geographics.com>
Committed: Fri Aug 9 19:37:52 2013 -0500
----------------------------------------------------------------------
doc/admin/configuring-cache.en.rst | 69 +--
doc/admin/getting-started.en.rst | 12 +-
doc/admin/http-proxy-caching.en.rst | 24 +-
doc/admin/index.en.rst | 2 +
doc/admin/security-options.en.rst | 88 ++--
doc/arch/cache/cache-api.en.rst | 25 ++
doc/arch/cache/cache-appendix.en.rst | 80 ++++
doc/arch/cache/cache-arch.en.rst | 441 +++++++++++++++++++
doc/arch/cache/cache-data-structures.en.rst | 158 +++++++
doc/arch/cache/cache-glossary.en.rst | 22 +
doc/arch/cache/cache.en.rst | 34 ++
doc/arch/cache/common.defs | 31 ++
.../images/ats-cache-doc-layout-pre-3-2-0.png | Bin 0 -> 7731 bytes
doc/arch/cache/images/ats-cache-doc-layout.png | Bin 0 -> 9470 bytes
doc/arch/cache/images/ats-cache-layout.jpg | Bin 0 -> 55045 bytes
.../cache/images/ats-cache-storage-units.png | Bin 0 -> 6190 bytes
.../images/ats-cache-volume-definition.png | Bin 0 -> 3086 bytes
.../cache/images/ats-cache-volume-directory.png | Bin 0 -> 34949 bytes
.../cache/images/ats-cache-volume-layout.png | Bin 0 -> 8471 bytes
.../cache/images/ats-cache-write-cursor.png | Bin 0 -> 11496 bytes
doc/arch/cache/tier-storage.en.rst | 57 +++
doc/arch/index.en.rst | 31 ++
doc/ext/traffic-server.py | 32 +-
doc/index.rst | 1 +
doc/reference/api/TSHttpHookAdd.en.rst | 19 +-
.../configuration/ip_allow.config.en.rst | 26 +-
.../configuration/records.config.en.rst | 20 +-
27 files changed, 1052 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/admin/configuring-cache.en.rst
----------------------------------------------------------------------
diff --git a/doc/admin/configuring-cache.en.rst b/doc/admin/configuring-cache.en.rst
index 287a9ee..2cf11b4 100644
--- a/doc/admin/configuring-cache.en.rst
+++ b/doc/admin/configuring-cache.en.rst
@@ -1,22 +1,24 @@
+.. _configuring-the-cache:
+
Configuring the Cache
*********************
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
The Traffic Server cache consists of a high-speed object database called
the **object store** that indexes objects according to URLs and their
@@ -92,8 +94,7 @@ To change the RAM cache size:
gigabyte of disk.
3. Restart Traffic Server. If you increase the RAM cache to a size or
1GB or more, then restart with the ``start_traffic_server`` command
- (refer to `Starting Traffic
- Server <../getting-started#StartingTS>`_).
+ (refer to :ref:`start-traffic-server`).
@@ -114,8 +115,7 @@ steps below:
1. Stop Traffic Server.
2. Add hardware, if necessary.
-3. Edit the Traffic Server
- ```storage.config`` <../configuration-files/storage.config>`_ file:
+3. Edit :file:`storage.config` to
increase the amount of disk space allocated to the cache on existing
disks or describe the new hardware you are adding.
4. Restart Traffic Server.
@@ -129,13 +129,12 @@ steps below:
1. Stop Traffic Server.
2. Remove hardware, if necessary.
-3. Edit the Traffic Server
- ```storage.config`` <../configuration-files/storage.config>`_ file:
+3. Edit :file:`storage.config` to
reduce the amount of disk space allocated to the cache on existing
disks or delete the reference to the hardware you're removing.
4. Restart Traffic Server.
-**IMPORTANT:** In the ``storage.config`` file, a formatted or raw disk
+**IMPORTANT:** In :file:`storage.config`, a formatted or raw disk
must be at least 128 MB.
Partitioning the Cache
@@ -303,6 +302,8 @@ Note: The procedure above only removes an object from a *specific*
Traffic Server cache. Users may still see the old (removed) content if
it was cached by intermediary caches or by the end-users' web browser.
+.. _inspecting-the-cache:
+
Inspecting the Cache
====================
@@ -310,27 +311,27 @@ Traffic Server provides a Cache Inspector utility that enables you to
view, delete, and invalidate URLs in the cache (HTTP only). The Cache
Inspector utility is a powerful tool that's capable of deleting *all*
the objects in your cache; therefore, make sure that only authorized
-administrators are allowed to access this utility, see `Controlling Host
-Access to Traffic
-Manager <../security-options#ControllingHostAccessTrafficManager>`_.
+administrators are allowed to access this utility, see :ref:`controlling-client-access-to-cache` and the ``@scr_ip`` option in :file:`remap.config`.
Accessing the Cache Inspector Utility
-------------------------------------
To access the Cache Inspector utility, follow the steps below:
-1. In the `:file:`records.config` <../configuration-files/records.config>`_
- file add the following variable:
-2. `*``CONFIG proxy.config.http_ui_enabled INT 1``* <../configuration-files/records.config#proxy.config.http_ui_enabled>`_
-3. To access the cache inspector in reverse proxy mode, you must add a
- remap rule to ``remap.config`` to expose the URL For example:
- ``map http://yourhost.com/myCI http://{cache} @action=allow @src_ip=corp_internal_address``
-4. From the Traffic Server ``bin`` directory, enter the following
+#. Set :ts:cv:`proxy.config.http_ui_enabled` to ``1``.
+#. To access the cache inspector in reverse proxy mode, you must add a
+ remap rule to :file:`remap.config` to expose the URL. This should be restricted to a limited set of hosts using the ``@src_ip`` option. To restrict access to the network 172.28.56.0/24, use ::
+
+ map http://yourhost.com/myCI http://{cache} @action=allow @src_ip=172.28.56.1-172.28.56.254
+
+#. From the Traffic Server ``bin`` directory, enter the following
command to re-read the configuration file: ``traffic_line -x``
-5. Open your web browser and configure it to use your Traffic Server as
- a proxy server. Type the following URL: ``http://yourhost/myCI``
-6. The Cache page opens (see `Using the Cache Page <#UsingCachePage>`_
- below).
+#. Open your web browser and configure it to use your Traffic Server as
+ a proxy server. Type the following URL::
+
+ http://yourhost/myCI
+
+#. The Cache page opens.
Using the Cache Page
--------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/admin/getting-started.en.rst
----------------------------------------------------------------------
diff --git a/doc/admin/getting-started.en.rst b/doc/admin/getting-started.en.rst
index a97c065..1cb5166 100644
--- a/doc/admin/getting-started.en.rst
+++ b/doc/admin/getting-started.en.rst
@@ -8,9 +8,9 @@ Getting Started
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -135,6 +135,8 @@ only work successfully with the default ``layout``:
After you have installed Traffic Server on your system, you can do any
of the following:
+.. _start-traffic-server:
+
Start Traffic Server
====================
@@ -143,11 +145,9 @@ passing in the attribute ``start``. This command starts all the
processes that work together to process Traffic Server requests as well
as manage, control, and monitor the health of the Traffic Server system.
-To run the ``trafficserver start`` command, e.g.:
-
-::
+To run the ``trafficserver start`` command, e.g.::
- bin/trafficserver start
+ bin/trafficserver start
At this point your server is up and running in the default configuration
of a `reverse proxy <../reverse-proxy-http-redirects>`_.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/admin/http-proxy-caching.en.rst
----------------------------------------------------------------------
diff --git a/doc/admin/http-proxy-caching.en.rst b/doc/admin/http-proxy-caching.en.rst
index b5e8153..c1f2b1b 100644
--- a/doc/admin/http-proxy-caching.en.rst
+++ b/doc/admin/http-proxy-caching.en.rst
@@ -1,22 +1,24 @@
+.. _http-proxy-caching:
+
HTTP Proxy Caching
******************
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
Web proxy caching enables you to store copies of frequently-accessed web
objects (such as documents, images, and articles) and then serve this
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/admin/index.en.rst
----------------------------------------------------------------------
diff --git a/doc/admin/index.en.rst b/doc/admin/index.en.rst
index 560b77b..f487071 100644
--- a/doc/admin/index.en.rst
+++ b/doc/admin/index.en.rst
@@ -1,3 +1,5 @@
+.. _admin-guide:
+
Administrators' Guide
**********************
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/admin/security-options.en.rst
----------------------------------------------------------------------
diff --git a/doc/admin/security-options.en.rst b/doc/admin/security-options.en.rst
index 2e3c56a..e9c3546 100644
--- a/doc/admin/security-options.en.rst
+++ b/doc/admin/security-options.en.rst
@@ -3,20 +3,20 @@ Security Options
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
Traffic Server provides a number of security features.
@@ -25,16 +25,17 @@ This chapter discusses the following topics:
.. toctree::
:maxdepth: 2
+.. _controlling-client-access-to-cache:
+
Controlling Client Access to the Proxy Cache
============================================
You can configure Traffic Server to allow only certain clients to use
the proxy cache by editing a configuration file.
-2. Add a line in the file ``ip_allow.config`` for each IP address or
- range of IP addresses allowed to access Traffic Server (refer to
- `ip_allow.config <configuration-files/ip_allow.config>`_).
-3. Run the command ``traffic_line -x`` to apply the configuration
+#. Add a line in :file:`ip_allow.config` for each IP address or
+ range of IP addresses allowed to access Traffic Server.
+#. Run the command ``traffic_line -x`` to apply the configuration
changes.
Configuring DNS Server Selection (Split DNS)
@@ -57,12 +58,10 @@ To configure Split DNS, you must do the following:
To do this, we
-2. Add rules to the ``splitdns.config`` file. (Refer to
- ```splitdns.config`` <../configuration-files/splitdns.config>`_.
-3. In the file :file:`records.config` the variable
- `*``proxy.process.dns.splitDNS.enabled``* <configuration-files/records.config#proxy.process.dns.splitDNS.enabled>`_
- to ``1`` to enable split DNS.
-4. Run the command ``traffic_line -x`` to apply the configuration
+#. Add rules to :file:`splitdns.config`.
+#. In :file:`records.config` set the variable
+ :ts:cv:`proxy.config.dns.splitDNS.enabled` to ``1`` to enable split DNS.
+#. Run the command ``traffic_line -x`` to apply the configuration
changes.
Using SSL Termination
@@ -141,9 +140,7 @@ client/Traffic Server connections, you must do the following:
authenticates the client. If you configure Traffic Server to *not*
require client certificates, then access to Traffic Server is
managed through other Traffic Server options that have been set
- (such as rules in the
- ```ip_allow.config`` <configuration-files/ip_allow.config>`_
- file).
+ (such as rules in :file:`ip_allow.config`).
- Specify the filename and location of the Traffic Server private
key (if the private key is not located in the server certificate
file). Traffic Server uses its private key during the SSL
@@ -156,16 +153,15 @@ client/Traffic Server connections, you must do the following:
In order to accomplish this, we
2. Edit the following variables in the ``SSL Termination`` section of
- the :file:`records.config` file:
+ :file:`records.config`
- - `*``proxy.config.ssl.enabled``* <configuration-files/records.config#proxy.config.ssl.enabled>`_
- - `*``proxy.config.ssl.server_port``* <configuration-files/records.config#proxy.config.ssl.server_port>`_
- - `*``proxy.config.ssl.client.certification_level``* <configuration-files/records.config#proxy.config.ssl.client.certification_level>`_
- - `*``proxy.config.ssl.server.cert.path``* <configuration-files/records.config#proxy.config.ssl.server.cert.path>`_
- - `*``proxy.config.ssl.server.private_key.filename``* <configuration-files/records.config#proxy.config.ssl.server.private_key.filename>`_
- - `*``proxy.config.ssl.server.private_key.path``* <configuration-files/records.config#proxy.config.ssl.server.private_key.path>`_
- - `*``proxy.config.ssl.CA.cert.filename``* <configuration-files/records.config#proxy.config.ssl.CA.cert.filename>`_
- - `*``proxy.config.ssl.CA.cert.path``* <configuration-files/records.config#proxy.config.ssl.CA.cert.path>`_
+ - :ts:cv:`proxy.config.http.server_ports`
+ - :ts:cv:`proxy.config.ssl.client.certification_level`
+ - :ts:cv:`proxy.config.ssl.server.cert.path`
+ - :ts:cv:`proxy.config.ssl.server.private_key.filename`
+ - :ts:cv:`proxy.config.ssl.server.private_key.path`
+ - :ts:cv:`proxy.config.ssl.CA.cert.filename`
+ - :ts:cv:`proxy.config.ssl.CA.cert.path`
3. Run the command ``traffic_line -L`` to restart Traffic Server on the
local node or ``traffic_line -M`` to restart Traffic Server on all
@@ -229,17 +225,17 @@ Traffic Server and origin server connections, you must do the following:
In order to accomplish this, we:
2. Edit the following variables in the ``SSL Termination`` section of
- the :file:`records.config` file:
-
- - `*``proxy.config.ssl.auth.enabled``* <configuration-files/records.config#proxy.config.ssl.auth.enabled>`_
- - ```proxy.config.ssl.server_port`` <configuration-files/records.config#proxy.config.ssl.server_port>`_
- - ```proxy.config.ssl.client.verify.server`` <configuration-files/records.config#proxy.config.ssl.client.verify.server>`_
- - ```proxy.config.ssl.client.cert.filename`` <configuration-files/records.config#proxy.config.ssl.client.cert.filename>`_
- - ```proxy.config.ssl.client.cert.path`` <configuration-files/records.config#proxy.config.ssl.client.cert.path>`_
- - ```proxy.config.ssl.client.private_key.filename`` <configuration-files/records.config#proxy.config.ssl.client.private_key.filename>`_
- - ```proxy.config.ssl.client.private_key.path`` <configuration-files/records.config#proxy.config.ssl.client.private_key.path>`_
- - ```proxy.config.ssl.client.CA.cert.filename`` <configuration-files/records.config#proxy.config.ssl.client.CA.cert.filename>`_
- - ```proxy.config.ssl.client.CA.cert.path`` <configuration-files/records.config#proxy.config.ssl.client.CA.cert.path>`_
+ :file:`records.config`:
+
+ - :ts:cv:`proxy.config.ssl.auth.enabled`
+ - :ts:cv:`proxy.config.http.server_ports`
+ - :ts:cv:`proxy.config.ssl.client.verify.server`
+ - :ts:cv:`proxy.config.ssl.client.cert.filename`
+ - :ts:cv:`proxy.config.ssl.client.cert.path`
+ - :ts:cv:`proxy.config.ssl.client.private_key.filename`
+ - :ts:cv:`proxy.config.ssl.client.private_key.path`
+ - :ts:cv:`proxy.config.ssl.client.CA.cert.filename`
+ - :ts:cv:`proxy.config.ssl.client.CA.cert.path`
3. Run the command ``traffic_line -L`` to restart Traffic Server on the
local node or ``traffic_line -M`` to restart Traffic Server on all
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/cache-api.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/cache-api.en.rst b/doc/arch/cache/cache-api.en.rst
new file mode 100755
index 0000000..7271160
--- /dev/null
+++ b/doc/arch/cache/cache-api.en.rst
@@ -0,0 +1,25 @@
+.. include:: common.defs
+
+===========================
+Cache Related API functions
+===========================
+
+.. c:function:: void TSHttpTxnReqCacheableSet(TSHttpTxn txnp, int flag)
+
+ Set a *flag* that marks a request as cacheable. This is a positive override only, setting *flag* to 0 restores the default behavior, it does not force the request to be uncacheable.
+
+.. c:function:: TSReturnCode TSCacheUrlSet(TSHttpTxn txnp, char const* url, int length)
+
+ Set the cache key for the transaction *txnp* as the string pointed at by *url* of *length* characters. It need not be ``null`` terminated. This should be called from ``TS_HTTP_READ_REQUEST_HDR_HOOK`` which is before cache lookup but late enough that the HTTP request header is available.
+
+===============
+Cache Internals
+===============
+
+.. cpp:function:: int DIR_SIZE_WITH_BLOCK(int big)
+
+ A preprocessor macro which computes the maximum size of a fragment based on the value of *big*. This is computed as if the argument where the value of the *big* field in a struct :cpp:class:`Dir`.
+
+.. cpp:function:: int DIR_BLOCK_SIZE(int big)
+
+ A preprocessor macro which computes the block size multiplier for a struct :cpp:class:`Dir` where *big* is the *big* field value.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/cache-appendix.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/cache-appendix.en.rst b/doc/arch/cache/cache-appendix.en.rst
new file mode 100755
index 0000000..39b8e79
--- /dev/null
+++ b/doc/arch/cache/cache-appendix.en.rst
@@ -0,0 +1,80 @@
+.. include:: common.defs
+
+Cache Tools
+~~~~~~~~~~~
+
+Tools and techniques for cache monitoring and inspection.
+
+* :ref:`The cache inspector <inspecting-the-cache>`.
+
+Topics to be done
+~~~~~~~~~~~~~~~~~
+
+* Resident alternates
+* Object refresh
+
+Cache Consistency
+~~~~~~~~~~~~~~~~~
+
+The cache is completely consistent, up to and including kicking the power cord out, if the write buffer on consumer disk drives is disabled. You need to use::
+
+ hdparm -W0
+
+The cache validates that all the data for the document is available and will silently mark a partial document as a "miss" on read. There is no "gentle" shutdown for traffic server, you just kill the process, so the "recovery" code (fsck) is run every time traffic server starts up.
+
+On startup the two versions of the index are checked, and the last valid one is read into memory. Then traffic server moves forward from the last snapped write cursor and reads all the fragments written to disk, and updates the directory (as in a log-based file system). It stops reading at the write before the last valid write header it sees (as a write is not necessarily atomic because of sector reordering). Then the new updated index is written to the invalid version (in case of a crash during startup) and the system starts.
+
+.. _volume tagging:
+
+Volume Tagging
+~~~~~~~~~~~~~~
+
+Currently cache volumes are allocated somewhat arbitrarily from storage elements. `This enhancement <https://issues.apache.org/jira/browse/TS-1728>`__ allows the :file:`storage.config` file to assign storage units to specific volumes although the volumes must still be listed in :file:`volume.config` in general and in particular to map domains to specific volumes. A primary use case for this is to be able to map specific types of content to different storage elements. This could to have different storage devices for the content (SSD vs. rotational).
+
+---------------
+Version Upgrade
+---------------
+
+It is currently the case that any change to the cache format will clear the cache. This is an issue when upgrading the |TS| version and should be kept in mind.
+
+.. cache-key:
+
+-------------------------
+Controlling the cache key
+-------------------------
+
+The cache key is by default the URL of the request. There are two possible choices, the original ("pristine") URL and the remapped URL. Which of these is used is determined by the configuration value :ts:cv:`proxy.config.url_remap.pristine_host_hdr`.
+
+This is an ``INT`` value. If set to ``0`` (disabled) then the remapped URL is used, and if it is not ``0`` (enabled) then the original URL is used. This setting also controls the value of the ``HOST`` header that is placed in the request sent to the origin server, using hostname from the original URL if non-``0`` and the host name from the remapped URL if ``0``. It has no other effects.
+
+For caching, this setting is irrelevant if no remapping is done or there is a one to one mapping between the original and remapped URLs.
+
+It becomes significant if multiple original URLs are mapped to the same remapped URL. If pristine headers are enabled requests to different original URLs will be stored as distinct objects in the cache. If disabled the remapped URL will be used and there may be collisions. This is bad if the contents different but quite useful if they are the same (e.g., the original URLs are just aliases for the same underlying server).
+
+This is also an issue if a remapping is changed because it is effectively a time axis version of the previous case. If an original URL is remapped to a different server address then the setting determines if existing cached objects will be served for new requests (enabled) or not (disabled). Similarly if the original URL mapped to a particular URL is changed then cached objects from the initial original URL will be served from the updated original URL if pristine headers is disabled.
+
+These collisions are not of themselves good or bad. An administrator needs to decide which is appropriate for his situation and set the value correspondingly.
+
+If a greater degree of control is desired a plugin must used to invoke the API call :c:func:`TSCacheUrlSet()` to provide a specific cache key.
+
+A plugin that changes the cache key *must* do so consistently for both cache hit and cache miss requests because two different requests that map to the same cache key will be considered equivalent by the cache. Use of the URL directly provides this and so must any substitute. This is entirely the responsibility of the plugin, there is no way for the |TS| core to detect such an occurrence.
+
+It is a requirement that the string be syntactically a URL but otherwise it is completely arbitrary and need not have any path. For instance if the company Network Geographics wanted to store certain content under its own cache key, using a document GUID as part of the key, it could use a cache key like ::
+
+ ngeo://W39WaGTPnvg
+
+The scheme ``ngeo`` was picked because it is *not* a valid URL scheme and so will not collide with any valid URL.
+
+This can be useful if the URL encodes both important and unimportant data. Instead of storing potentially identical content under different URLs (because they differ on the unimportant parts) a url containing only the important parts could be created and used.
+
+For example, suppose the URL for Network Geographics content encoded both the document GUID and a referral key. ::
+
+ http://network-geographics-farm-1.com/doc/W39WaGTPnvg.2511635.UQB_zCc8B8H
+
+We don't want to the same content for every possible referrer. Instead we could use a plugin to convert this to the previous example and requests that differed only in the referrer key would all reference the same cache entry. Note that we would also map ::
+
+ http://network-geographics-farm-56.com/doc/W39WaGTPnvg.2511635.UQB_zCc8B8H
+
+to the same cache key. This can be handy for "sharing" content between servers when that content is identical. Note also the plugin can change the cache key or not depending on any data in the request header, for instance not changing the cache key if the request is not in the ``doc`` directory. If distinguishing servers is important that can easily be pulled from the request URL and used in the synthetic cache key. The implementor is free to extract all relevant elements for use in the cache key.
+
+While there is explicit no requirement that the synthetic cache key be based on the HTTP request header, in practice it is generally necessary due to the consistency requirement. Because cache lookup happens before attempting to connect to the origin server no data from the HTTP response header is available, leaving only the request header. The most common case is the one described above where the goal is to elide elements of the URL that do not affect the content to minimize cache footprint and improve cache hit rates.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/cache-arch.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/cache-arch.en.rst b/doc/arch/cache/cache-arch.en.rst
new file mode 100755
index 0000000..d2dbc3c
--- /dev/null
+++ b/doc/arch/cache/cache-arch.en.rst
@@ -0,0 +1,441 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Cache Architecture
+******************
+
+.. include:: common.defs
+
+Introduction
+~~~~~~~~~~~~
+
+In addition to an HTTP proxy, |ATS| is also an HTTP cache. |TS| can cache any octet stream although it currently supports only those octet streams delivered by the HTTP protocol. When such a stream is cached it is termed an *object* in the cache. The server from which the content was retrieved is called the *origin server*. A stored object contains the octet stream content and the HTTP request and response headers. Each object is identified by a globally unique value called a *cache key*. By default this is generated by taking the `MD5 hash <http://www.openssl.org/docs/crypto/md5.html>`_ of the URI used to retrieve the content from the origin server.
+
+The purpose of this document is to provide implementation details of the |TS| cache. Configuration of the cache will be discussed only to the extent needed to understand the internal mechanisms. This documeint will be useful primarily to |TS| developers working on the |TS| codebase or plugins for |TS|. It is assumed the reader is already familiar with the :ref:`admin-guide` and specifically with :ref:`http-proxy-caching` and :ref:`configuring-the-cache`, along with the associated configuration files and values.
+
+Cache Layout
+~~~~~~~~~~~~
+
+The first step in understanding cache operations is to understand the data structures and layout of the cache.
+
+Cache storage
+=============
+
+The raw storage for the |TS| cache is configured in :file:`storage.config`. This defines the "storage units" used by the cache. The storage units are treated as undifferentiated byte ranges.
+
+.. figure:: images/ats-cache-storage-units.png
+ :align: center
+
+ Example cache storage units
+
+This storage is used by defining *volumes* in :file:`volume.config`. Volumes can be defined by a percentage of the total storage or an absolute amount of storage. Each volume is striped across the storage units so that every volume will be on every storage unit for robustness and performance.
+
+If the volumes for the example storage units were defined as
+
+.. image:: images/ats-cache-volume-definition.png
+ :align: center
+
+then the actual layout would look like
+
+.. image:: images/ats-cache-volume-layout.png
+ :align: center
+
+A cached object is stored entirely in a single volume and a single storage unit, objects are never split across volumes. The storage that holds a single object is called a *document*. Objects are assigned to volumes automatically based on a hash of the URI used to retrieve the object from the origin server. It is possible to configure this to a limited extent in :file:`hosting.config` which supports content from specific host or domain to be stored on specific volumes.
+
+Volume Structure
+================
+
+Volumes are treated as an undifferentiated span of bytes. Internally each stripe on each storage unit is treated almost entirely independently. The data structures described in this section are duplicated for each volume stripe, the part of a volume that resides in a single storage unit. This is how the term "volume" and :cpp:class:`Vol` are used inside the code. What a user thinks of as a volume of the cache is stored in the little used :cpp:class:`CacheVol`.
+
+.. index: write cursor
+.. _write-cursor:
+
+Each storage unit in a volume is divided in to two areas -- content and directory. The directory area is used to maintain disk backups of the :ref:`in memory directory <volume-directory>`. The content area stores the actual objects and is used as a circular buffer where new documents overwrite the least recently cached documents. In particular no operating system file structure is present inside a cache volume. The location in a volume where new cache data is written is called the *write cursor*. This means that objects can be de facto evicted from cache even if they have not expired if the data is overwritten by the write cursor.
+
+.. figure:: images/ats-cache-write-cursor.png
+ :align: center
+
+ The write cursor and documents in the cache.
+
+.. index:: volume directory
+.. _volume-directory:
+
+To find objects each volume has a directory of all documents in that volume. This directory is kept memory resident which means cache misses do not cause disk I/O. A side effect of this is that increasing the size of the cache (not storing more objects) increases the memory footprint of Traffic Server. Every document consumes at least one directory entry, although larger documents can require more entries.
+
+.. index:: cache key
+.. _cache-key:
+
+The directory is a hash table with a 128 bit key. This kind of key is referred to as a *cache key*. The cache key for an object is used to locate the corresponding directory entry after volume assignment [#]_. This entry in turn references a span in the volume content area which contains the object header and possibly the object as well. The size stored in the directory entry is an :ref:`approximate size <dir-size>` which is at least as big as the actual data on disk. The document header on disk contains metadata for the document including the exact size of the entire document, and the HTTP headers associated with the object.
+
+.. note:: Data in HTTP headers cannot be examined without disk I/O. This includes the original URL for the object, as only the cache key (possibly) derived from it is stored in memory.
+
+For persistence the directory is stored on disk in copies (A and B), one of which is "clean" and the other of which is being written from memory. These are stored in the directory section of the volume.
+
+.. figure:: images/ats-cache-volume-directory.png
+ :align: center
+
+ Volume directory structure
+
+The total size of the directory (the number of entries) is computed by taking the size of the volume and dividing by the average object size. The directory always consumes this amount of memory which has the effect that if cache size is increased so is the memory requirement for |TS|. The average object size defaults to 8000 bytes but can be configured using the value::
+
+ proxy.config.cache.min_average_object_size
+
+in :file:`records.config`. Increasing the average object size will reduce the memory footprint of the directory at the expense of reducing the number of distinct objects that can be stored in the cache [#]_.
+
+.. note:: Cache data on disk is never updated.
+
+This is a key thing to keep in mind. What appear to be updates (such as doing a refresh on stale content and getting back a 304) are actually new copies of data being written at the write cursor. The originals are left as "dead" space which will be consumed when the write cursor arrives at that disk location. Once the volume directory is updated (in memory!) the original object in the cache is effectively destroyed. This is the general space management techinque used in other cases as well. If an object needs to removed from cache, only its volume directory entry is changed. No other work (and *particulary* no disk I/O) needs to be done.
+
+.. [#] Because each storage unit in each volume has a separate directory, the assignment must be done before the directory lookup.
+
+.. [#] An interesting potential optimization would be configuring average object size per cache volume.
+
+Object Structure
+================
+
+Objects are stored as two types of data, metadata and content data. Metadata is all the data about the object and the content and includes the HTTP headers. The content data is the content of the object, the actual data delivered to the client as the object.
+
+Objects are rooted in a :cpp:class:Doc structure stored in the cache. This is termed the "first ``Doc``" and always contains the metadata. It is always accessed first for any object. This ``Doc`` is located by doing a lookup of the corresponding cache key in the volume directory which specifies the location and approximate size of the ``Doc``. The ``Doc`` itself has fully accurate size data of both that specific ``Doc`` and the object.
+
+.. index:: alternate
+
+|TS| supports `varying content <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.44>`_ for objects. These are called *alternates*. All metadata for all alternates is stored in the first ``Doc`` including the set of alternates and the HTTP headers for them. This enables `alternate selection <http://trafficserver.apache.org/docs/trunk/sdk/http-hooks-and-transactions/http-alternate-selection.en.html>`_ to be done after the initial read from disk. An object that has more than one alternate will have the alternate content stored separately from the first ``Doc``. For objects with only one alternate the content may or may not be in the same (first) fragment as the metadata. Each separate alternate content is allocated a volume directory entry and the key for that entry is stored in the first ``Doc`` metadata.
+
+Prior to version 3.2.0 the header data was stored in the :cpp:class:`CacheHTTPInfoVector` class which was marshaled to a variable length area of the on disk image.
+
+.. figure:: images/ats-cache-doc-layout-pre-3-2-0.png
+ :align: center
+
+ ``Doc`` layout, pre 3.2.0
+
+This had the problem that with only one fragment table it could not be reliable accurate for objects with more than one alternate [#]_. Therefore the fragment data was moved from being a separate variable length section of the metadata to being directly incorporated in to the :cpp:class:`CacheHTTPInfoVector`, yielding a layout of the following form.
+
+.. figure:: images/ats-cache-doc-layout.png
+ :align: center
+
+ ``Doc`` layout, 3.2.0
+
+Each element in the vector contains for each alternate, in addition to the HTTP headers and the fragment table (if any), a cache key. This cache key identifies a volume directory entry that is referred to as the "earliest ``Doc``". This is the location where the content for the alternate begins.
+
+When the object is first cached, it will have a single alternate and that will be stored (if not too large) in first ``Doc``. This is termed a *resident alternate* in the code. Resident alternates are not liked and the next time the header information is updated the object content will be separated.
+
+.. note:: The :cpp:class:`CacheHTTPInfoVector` is stored only in the first ``Doc``. Subsequent ``Doc`` instances will have an ``hlen`` of zero.
+
+Large objects are split in to *fragments* when written to the cache. Each fragment has its own entry in the volume directory. This is indicated by a total document length that is longer than the content in first ``Doc`` or an earliest ``Doc``. In such a case a fragment offset table is stored. This contains the byte offset in the object content of the first byte of content data for each fragment past the first (as the offset for the first is always zero). This allows range requests to be serviced much more efficiently for large objects, as intermediate fragments can be skipped the first fragment with relevant data loaded next after the first/earliest ``Doc``. The last fragment in the sequence is detected by the fragment size and offset reaching the end of the total size of the object, there is no explicit end mark. Each fragment is computationally chained from the previous in that the cache key for fragment N is computed by::
+
+ key_for_N_plus_one = next_key(key_for_N);
+
+where ``next_key`` is a global function that deterministically computes a new cache key from an existing cache key.
+
+Objects with multiple fragments are laid out such that the data fragments (including the earliest ``Doc``) are written first and the first ``Doc`` is written last. When read from disk, both the first and earliest ``Doc`` are validated (tested to ensure that they haven't been overwritten by the write cursor) to verify that the entire document is present on disk (as they bookend the other fragments - the write cursor cannot overwrite them without overwriting at leastone of the verified ``Doc`` instances). Note that while the fragments of a single object are ordered they are not necessarily contiguous as data from different objects are interleaved as the data arrives in |TS|.
+
+.. index:: pinned
+
+Documents which are "pinned" into the cache must not be overwritten so they are "evacuated" from in front of the write cursor. Each fragment is read and rewritten. There is a special lookup mechanism for objects that are being evacuated so that they can be found in memory rather than the potentially unreliable disk regions. The cache scans ahead of the write cursor to discover pinned objects as there is a dead zone immediately before the write cursor from which data cannot be evacuated. Evacuated data is read from disk and placed in the write queue and written as its turn comes up.
+
+It appears that objects can only be pinned via the :file:`cache.config` file and if the value::
+
+ proxy.config.cache.permit.pinning
+
+is set to non-zero (it is zero by default). Objects which are in use when the write cursor is near use the same underlying evacuation mechanism but are handled automatically and not via the explicit ``pinned`` bit in :cpp:class:`Dir`.
+
+.. [#] It could, under certain circumstances, be accurate for none of the alternates.
+
+Additional Notes
+====================
+
+Some general observations on the data structures.
+
+Cyclone buffer
+--------------
+
+Because the cache is a cyclone cache objects are not preserved for an indefinite time. Even if the object is not stale it can be overwritten as the cache cycles through its volume. Marking an object as ``pinned`` preserves the object through the passage of the write cursor but this is done by copying the object across the gap, in effect re-storing it in the cache. Pinning large objects or a large number objects can lead to a excessive disk activity. The original purpose of pinning seems to have been for small, frequently used objects explicitly marked by the administrator.
+
+This means the purpose of expiration data on objects is simply to prevent them from being served to clients. They are not in the standard sense deleted or cleaned up. The space can't be immediately reclaimed in any event because writing only happens at the write cursor. Deleting an object consists only of removing the directory entries in the volume directory which suffices to (eventually) free the space and render the document inaccessible.
+
+Historically the cache is designed this way because web content was relatively small and not particularly consistent. The design also provides high performance and low consistency requirements. There are no fragmentation issues for the storage, and both cache misses and object deletions require no disk I/O. It does not deal particularly well with long term storage of large objects. See the :ref:`volume tagging` appendix for details on some work in this area.
+
+Disk Failure
+------------
+
+The cache is designed to be relatively resistant to disk failures. Because each storage unit in each volume is mostly independent the loss of a disk simply means that the corresponding :cpp:class:`Vol` instances (one per cache volume that uses the storage unit) becomes unusable. The primary issue is updating the volume assignment table to both preserve assignments for objects on still operational volumes while distributing the assignments from the failed disk to those operational volumes. This mostly done in::
+
+ AIO_Callback_handler::handle_disk_failure
+
+Restoring a disk to active duty is quite a bit more difficult task. Changing the volume assignment of a cache key renders any currently cached data inaccessible. This is obviouly not a problem when a disk has failed, but is a bit trickier to decide which cached objects are to be de facto evicted if a new storage unit is added to a running system. The mechanism for this, if any, is still under investigation.
+
+Implementation Details
+======================
+
+Volume Directory
+----------------
+
+.. _directory-entry:
+
+The in memory volume directory entries are defined as described below.
+
+.. cpp:class:: Dir
+
+ Defined in |P-CacheDir.h|_.
+
+ =========== =================== ===================================================
+ Name Type Use
+ =========== =================== ===================================================
+ offset unsigned int:24 Offset of first byte of metadata (volume relative)
+ big unsigned in:2 Offset multiplier
+ size unsigned int:6 Size
+ tag unsigned int:12 Partial key (fast collision check)
+ phase unsigned int:1 Unknown
+ head unsigned int:1 Flag: first segment in a document
+ pinned unsigned int:1 Flag: document is pinned
+ token unsigned int:1 Flag: Unknown
+ next unsigned int:16 Segment local index of next entry.
+ offset_high inku16 High order offset bits
+ =========== =================== ===================================================
+
+ The volume directory is an array of ``Dir`` instances. Each entry refers to a span in the volume which contains a cached object. Because every object in the cache has at least one directory entry this data has been made as small as possible.
+
+ The offset value is the starting byte of the object in the volume. It is 40 bits long split between the *offset* (lower 24 bits) and *offset_high* (upper 16 bits) members. Note that since there is a directory for every storage unit in a cache volume, this is the offset in to the slice of a storage unit attached to that volume.
+
+.. _dir-size:
+
+ The *size* and *big* values are used to calculate the approximate size of the span which contains the object. This value is used as the number of bytes to read from storage at the offset value. The exact size is contained in the object metadata in :cpp:class:`Doc` which is consulted once the read has completed. For this reason the approximate size needs to be at least as large as the actual size but can be larger, at the cost of reading the extraneous bytes.
+
+ The computation of the approximate size of the fragment is defined as::
+
+ ( *size* + 1 ) * 2 ^ ( ``CACHE_BLOCK_SHIFT`` + 3 * *big* )
+
+ where ``CACHE_BLOCK_SHIFT`` is the bit width of the size of a basic cache block (9, corresponding to a sector size of 512). Therefore the value with current defines is::
+
+ ( *size* + 1 ) * 2 ^ (9 + 3 * *big*)
+
+ Because *big* is 2 bits the values for the multiplier of *size* are
+
+ .. _big-mult:
+
+ ===== =============== ===============
+ *big* Multiplier Maximum Size
+ ===== =============== ===============
+ 0 512 (2^9) 32768 (2^15)
+ 1 4096 (2^12) 262144 (2^18)
+ 2 32768 (2^15) 2097152 (2^21)
+ 3 262144 (2^18) 16777216 (2^24)
+ ===== =============== ===============
+
+ Note also that *size* is effectively offset by one, so a value of 0 indicates a single unit of the multiplier.
+
+.. _target-fragment-size:
+
+The target fragment size can set with the :file:`records.config` value
+
+ ``proxy.config.cache.target_fragment_size``
+
+This value should be chosen so that it is a multiple of a :ref:`cache entry multiplier <big-mult>`. It is not necessary to make it a power of 2 [#]_. Larger fragments increase I/O efficiency but lead to more wasted space. The default size (1M, 2^20) is a reasonable choice in most circumstances altough in very specific cases there can be benefit from tuning this parameter. |TS| imposes an internal maximum of a 4194232 bytes which is 4M (2^22) less the size of a struct :cpp:class:`Doc`. In practice then the largest reasonable target fragment size is 4M - 262144 = 3932160.
+
+When a fragment is stored to disk the size data in the cache index entry is set to the finest granularity permitted by the size of the fragment. To determine this consult the :ref:`cache entry multipler <big-mult>` table, find the smallest maximum size that is at least as large as the fragment. That will indicate the value of *big* selected and therefore the granularity of the approximate size. That represents the largest possible amount of wasted disk I/O when the fragment is read from disk.
+
+.. note:: The cache index entry size is used only for reading the fragment from disk. The actual size on disk, and the amount of cache space consumed, is the actual size of the content rounded up to the disk sector size (default 512 bytes).
+
+.. index:: DIR_DEPTH, index segment, index buckets
+
+The set of index entries for a volume are grouped in to *segments*. The number of segments for an index is selected so that there are as few segments as possible such that no segment has more than 2^16 entries. Intra-segment references can therefore use a 16 bit value to refer to any other entry in the segment.
+
+Index entries in a segment are grouped *buckets* each of ``DIR_DEPTH`` (currently 4) entries. These are handled in the standard hash table way, giving somewhat less than 2^14 buckets per segment.
+
+Object Metadata
+---------------
+
+The metadata for an object is stored in a :cpp:class:`Doc`.
+
+.. [#] The comment in :file:`records.config` is simply wrong.
+
+----------------
+Cache Operations
+----------------
+
+Cache activity starts after the HTTP request header has been parsed and remapped. Tunneled transactions do not interact with the cache because the headers are never parsed.
+
+To understand the logic we must introduce the term "cache valid" which means something that is directly related to an object that is valid to be put in the cache (e.g. a ``DELETE`` which refers to a URL that is cache valid but cannot be cached itself). This is important because |TS| computes cache validity several times during a transaction and only performs cache operations for cache valid results. The criteria used changes during the course of the transaction as well. This is done to avoid the cost of cache activity for objects that cannot be in the cache.
+
+The three basic cache operations are lookup, read, and write. We will take deleting entries as a special case of writing where only the volume directory is updated.
+
+After the client request header is parsed and is determined to be potentially cacheable, a `cache lookup`_ is done. If successful a `cache read`_ is attempted. If either the lookup or the read fails and the content is considered cacheable then a `cache write`_ is attempted.
+
+Cacheability
+============
+
+The first thing done with a request with respect to cache is to determine whether it is potentially a valid object for the cache. After initial parsing and remapping this check is done primarily to detect a negative result because if so all further cache processing is skipped -- it will not be put in to the cache nor will a cache lookup be done. There are a number of prerequisites along with configuration options to change them. Additional cacheability checks are done later in the process when more is known about the transaction (such as plugin operations and the origin server response). Those checks are described as appropriate in the sections on the relevant operations.
+
+The set of things which can affect cacheability are
+
+* Built in constraints
+* Settings in :file:`records.config`
+* Settings in :file:`cache.config`
+* Plugin operations
+
+The initial internal checks, along with their :file:`records.config` overrides[#]_, are done in::
+
+ HttpTransact::is_request_cache_lookupable
+
+The checks that are done are
+
+ Cacheable Method
+ The request must be one of ``GET``, ``HEAD``, ``POST``, ``DELETE``, ``PUT``.
+
+ See ``HttpTransact::is_method_cache_lookupable()``.
+
+ Dynamic URL
+ |TS| tries to avoid caching dynamic content because it's dynamic. A URL is considered dynamic if it
+
+ * is not ``HTTP`` or ``HTTPS``
+ * has query parameters
+ * ends in ``asp``
+ * has ``cgi`` in the path
+
+ This check can be disabled by setting a non-zero value for::
+
+ proxy.config.http.cache.cache_urls_that_look_dynamic
+
+ In addition if a TTL is set for rule that matches in :file:`cache.config` then this check is not done.
+
+ Range Request
+ Cache valid only if :ts:cv:`proxy.config.http.cache.range.lookup` in :file:`records.config` is non-zero. This does not mean the range request can be cached, only that it might be satisfiable from the cache.
+
+A plugin can call :c:func:`TSHttpTxnReqCacheableSet()` to force the request to be viewed as cache valid.
+
+.. [#] The code appears to check :file:`cache.config` in this logic by setting the ``does_config_permit_lookup`` in the ``cache_info.directives`` of the state machine instance but I can find no place where the value is used. The directive ``does_config_permit_storing`` is set and later checked so the directive (from the administrator point of view) is effective in preventing caching of the object.
+
+Cache Lookup
+============
+
+If the initial request is not determined to be cache invalid then a lookup is done. Cache lookup determines if an object is in the cache and if so, where it is located. In some cases the lookup proceeds to read the first ``Doc`` from disk to verify the object is still present in the cache.
+
+There are three basic steps to a cache lookup.
+
+#. The cache key is computed.
+
+ This is normally computed using the request URL but it can be overridden :ref:`by a plugin <cache-key>` . As far as I can tell the cache index string is not stored anywhere, it presumed computable from the client request header.
+
+#. The cache volume is determined (based on the cache key).
+
+ The cache key is used as a hash key in to an array of :cpp:class:`Vol` instances. The construction and arrangement of this array is the essence of how volumes are assigned.
+
+#. The cache volume directory is probed using the index key computed from the cache key.
+
+ Various other lookaside directories are checked as well, such as the :ref:`aggregation buffer <aggregation-buffer>`.
+
+#. If the directory entry is found the first ``Doc`` is read from disk and checked for validity.
+
+ This is done in :cpp:func:`CacheVC::openReadStartHead()` or :cpp:func:`CacheVC::openReadStartEarliest()` which are tightly coupled methods.
+
+If the lookup succeeds then a more detailed directory entry (struct :cpp:class:`OpenDir`) is created. Note that the directory probe includes a check for an already extant ``OpenDir`` which if found is returned without additional work.
+
+Cache Read
+==========
+
+Cache read starts after a successful `cache lookup`_. At this point the first ``Doc`` has been loaded in to memory and can be consulted for additional information. This will always contain the HTTP headers for all alternates of the object.
+
+.. sidebar:: Read while write
+
+ There is provision in the code to support "read while write", that is serving an object from cache in one transaction while it is being written in another. It is unclear to me if this actually works. It must specifically enabled in :file:`records.config` and if not, a cache read will fail if the object is currently be written or updated.
+
+At this point an alternate for the object is selected. This is done by comparing the client request to the stored response headers, but it can be controlled by a plugin using ``TS_HTTP_ALT_SELECT_HOOK``.
+
+The content can now be checked to see if it is stale by calculating the "freshness" of the object. This is essential checking how old the object is by looking at the headers and possibly other metadata (note the headers can't be checked until we've selected an alternate).
+
+Most of this work is done in::
+
+ HttpTransact::what_is_document_freshness
+
+First the TTL (time to live) value which can be set in:file:`cache.config` is checked if the request matches the configuration file line. This is done based on when the object was placed in cache, not on any data in the headers.
+
+Next an internal flag ("needs-revalidate-once") is checked if the :file:`cache.config` value "revalidate-after" is not set, and if set the object is marked "stale".
+
+After these checks the object age is calculated by::
+
+ HttpTransactHeaders::calculate_document_age
+
+and then any configured fuzzing is applied. The limits to this age based on available data is calculated by::
+
+ HttpTransact::calculate_document_freshness_limit
+
+How this age is used is determined by the :file:`records.config` value::
+
+ proxy.config.http.cache.when_to_revalidate
+
+If this is zero then the built caclulations are used which compare the freshness limits with document age, modified by any of the client supplied cache control values ``max-age``, ``min-fresh``, ``max-stale`` unless explicitly overridden in :file:`cache.config`.
+
+If the object is not stale then it is served to the client. If stale the client request may be changed to an ``If Modified Since`` request to revalidate.
+
+The request is served using a standard virtual connection tunnel (``HttpTunnel``) with the :cpp:class:`CacheVC` acting as the producer and the client ``NetVC`` acting as the sink. If the request is a range request this can be modified with a transform to select the appropriate parts of the object or, if the request contains a single range, it can use the range acceleration.
+
+Range acceleration is done by consulting a fragment offset table attached to the earliest ``Doc`` which contains offsets for all fragments past the first. This allows loading the fragment containing the first requested byte immediately rather than performing reads on the intermediate fragments.
+
+Cache Write
+===========
+
+Writing to cache is handled by an instance of the class :cpp:class:`CacheVC`. This is a virtual connection which receives data and writes it to cache, acting as a sink. For a standard transaction data transfers between virtual connections (*VConns*) are handled by :ccp:class:HttpTunnel. Writing to cache is done by attaching a ``CacheVC`` instance as a tunnel consumer. It therefore operates in parallel with the virtual connection that transfers data to the client. The data does not flow to the cache and then to the client, it is split and goes both directions in parallel. This avoids any data synchronization issues between the two.
+
+.. sidebar:: Writing to disk
+
+ The actual write to disk is handled in a separate thread dedicated to I/O operations, the AIO threads. The cache logic marshals the data and then hands the operation off to the AIO thread which signals back once the operation completes.
+
+While each ``CacheVC`` handles its transactions independently, they do interact at the volume level as each ``CacheVC`` makes calls to the volume object to write its data to the volume content. The ``CacheVC`` accumulates data internally until either the transaction is complete or the amount of data to write exceeds the target fragment size. In the former case the entire object is submitted to the volume to be written. In the latter case a target fragment size amount of data is submitted and the ``CacheVC`` continues to operate on subsequent data. The volume in turn places these write requests in an holding area called the `aggregation buffer`_.
+
+For objects under the target fragment size there is no consideration of order, the object is simply written to the volume content. For larger objects the earliest ``Doc`` is written first and the first ``Doc`` written last. This provides some detection ability should the object be overwritten. Because of the nature of the write cursor no fragment after the first fragment (in the earliest ``Doc``) can be overwritten without also overwriting that first fragment (since we know at the time the object was finalized in the cache the write cursor was at the position of the first ``Doc``).
+
+.. note:: It is the responsibility of the ``CacheVC`` to not submit writes that exceed the target fragment size.
+
+.. how does the write logic know if it's an original object write or an update to an existing object?
+
+Update
+------
+
+Cache write also covers the case where an existing object in the cache is modified. This occurs when
+
+* A conditional request is made to the origin server and a ``304 - Not Modified`` response is received.
+* An alternate of the object is retrieved from an origin server and added to the object.
+* An alternate of the object is removed (e.g., due to a ``DELETE`` request).
+
+In every case the metadata for the object must be modified. Because |TS| never updates data already in the cache this means the first ``Doc`` will be written to the cache again and the volume directory entry updated. Because a client request has already been processed the first ``Doc`` has been read from cache and is in memory. The alternate vector is updated as appropriate (an entry added or removed, or changed to contain the new HTTP headers), and then written to disk. It is possible for multiple alternates to be updated by different ``CacheVC`` instances at the same time. The only contention is the first ``Doc``, the rest of the data for each alternate is completely independent.
+
+.. _aggregation-buffer:
+
+Aggregation Buffer
+------------------
+
+Disk writes to cache are handled through an *aggregation buffer*. There is one for each :cpp:class:`Vol` instance.
+To minimize the number of system calls data is written to disk in units of roughly :ref:`target fragment size <target-fragment-size>` bytes. The algorithm used is simple - data is piled up in the aggregation buffer until no more will fit without going over the targer fragment size, at which point the buffer is written to disk and the volume directory entries for objects with data in the buffer are updated with the actual disk locations for those objects (which are determined by the write to disk action). After the buffer is written it is cleared and process repeats. There is a special lookup table for the aggregation buffer so that object lookup can find cache data in that memory.
+
+Because data in the aggregation buffer is visible to other parts of the cache, particularly `cache lookup`_, there is no need to push a partial filled aggregation buffer to disk. In effect any such data is effectively memory cached until enough additional cache content arrives to fill the buffer.
+
+The target fragment size has little effect on small objects because the fragment sized is used only to parcel out disk write operations. For larger objects the effect very significant as it causes those objects to be broken up in to fragments at different locations on in the volume. Each fragment write has its own entry in the volume directory which are computational chained (each cache key is computed from the previous one). If possible a fragment table is accumulated in the earliest ``Doc`` which has the offsets of the first byte for each fragment.
+
+Evacuation
+----------
+
+By default the write cursor will overwrite (de facto evict from cache) objects as it proceeds once it has gone around the volume content at least once. In some cases this is not acceptable and the object is *evacuated* by reading it from the cache and then writing it back to cache which moves the physical storage of the object from in front of the write cursor to behind the write cursor. Objects that are evacuated are those that are active in either a read or write operation, or objects that are pinned [#]_.
+
+Evacuation starts by dividing up the volume content in to a set of regions of ``EVACUATION_BUCKET_SIZE`` bytes. The :cpp:member:`Vol::evacuate` member is an array with an element for each region. Each element is a doubly linked list of :cpp:class:`EvacuationBlock` instances. Each instance contains a :cpp:class:`Dir` that specifies the document to evacuate. Objects to be evacuated are descrinbed in an ``EvacuationBlock`` which is put in to an evacuation bucket based on the offset of the storage location.
+
+There are two types of evacuations, reader based and forced. The ``EvacuationBlock`` has a reader count to track this. If the reader count is zero, then it is a forced evacuation and the the target, if it exists, will be evacuated when the write cursor gets close. If the reader value is non-zero then it is a count of entities that are currently expecting to be able to read the object. Readers increment the count when they require read access to the object, or create the ``EvacuationBlock`` with a count of 1. When a reader is finished with the object it decrements the count and removes the ``EvacuationBlock`` if the count goes to zero. If the ``EvacuationBlock`` already exists with a count of zero, the count is not modified and the number of readers is not tracked, so the evacuation be valid as long as the object exists.
+
+Objects are evacuated as the write cursor approaches. The volume calculates the current amount of
+
+Before doing a write, the method :cpp:func:`Vol::evac_range()` is called to start an evacuation. If an eva
+
+.. [#] `Work is under way <https://issues.apache.org/jira/browse/TS-2020>`_ on extending this to include objects that are in the ram cache.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/cache-data-structures.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/cache-data-structures.en.rst b/doc/arch/cache/cache-data-structures.en.rst
new file mode 100755
index 0000000..57a3467
--- /dev/null
+++ b/doc/arch/cache/cache-data-structures.en.rst
@@ -0,0 +1,158 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Cache Data Structures
+******************************
+
+.. include:: common.defs
+
+.. cpp:class:: OpenDir
+
+ An open directory entry. It contains all the information of a :cpp:class:`Dir` plus additional information from the first :cpp:class:`Doc`.
+
+.. cpp:class:: CacheVC
+
+ A virtual connection class which accepts input for writing to cache.
+
+.. cpp:function:: int CacheVC::openReadStartHead(int event, Event* e)
+
+ Do the initial read for a cached object.
+
+.. cpp:function:: int CacheVC::openReadStartEarliest(int event, Event* e)
+
+ Do the initial read for an alternate of an object.
+
+.. cpp:class:: HttpTunnel
+
+ Data transfer driver. This contains a set of *producers*. Each producer is connected to one or more *consumers*. The tunnel handles events and buffers so that data moves from producers to consumers. The data, as much as possible, is kept in reference counted buffers so that copies are done only when the data is modified or for sources (which acquire data from outside |TS|) and sinks (which move data to outside |TS|).
+
+.. cpp:class:: CacheControlResult
+
+ Holds the data from a line in :file:`cache.config`.
+
+.. cpp:class:: CacheHTTPInfoVector
+
+ Defined in |P-CacheHttp.h|_. This is an array of :cpp:class:`HTTPInfo` objects and serves as the respository of information about alternates of an object. It is marshaled as part of the metadata for an object in the cache.
+
+.. cpp:class:: HTTPInfo
+
+ Defined in |HTTP.h|_.
+
+ This class is a wrapper for :cpp:class:`HTTPCacheAlt`. It provides the external API for accessing data in the wrapped class. It contains only a pointer (possibly ``NULL``) to an instance of the wrapped class.
+
+.. cpp:class:: CacheHTTPInfo
+
+ A typedef for :cpp:class:`HTTPInfo`.
+
+.. cpp:class:: HTTPCacheAlt
+
+ Defined in |HTTP.h|_.
+
+ This is the metadata for a single alternate for a cached object. In contains among other data
+
+ * The key for the earliest ``Doc`` of the alternate.
+ * The request and response headers.
+ * The fragment offset table. [#]_
+ * Timestamps for request and response from origin server.
+
+.. cpp:class:: EvacuationBlock
+
+ Record for evacuation.
+
+.. cpp:class:: Vol
+
+ This represents a storage unit inside a cache volume.
+
+.. cpp:member:: off_t Vol::segments
+
+ The number of segments in the volume. This will be roughly the total number of entries divided by the number of entries in a segment. It will be rounded up to cover all entries.
+
+.. cpp:member:: off_t Vol::buckets
+
+ The number of buckets in the volume. This will be roughly the number of entries in a segment divided by ``DIR_DEPTH``. For currently defined values this is around 16,384 (2^16 / 4). Buckets are used as the targets of the index hash.
+
+.. cpp:member:: DLL\<EvacuationBlock\> Vol::evacuate
+
+ Array of of :cpp:class:`EvacuationBlock` buckets. This is sized so there is one bucket for every evacuation span.
+
+.. cpp:function:: int Vol::evac_range(off_t low, off_t high, int evac_phase)
+
+ Start an evacuation if there is any :cpp:class:`EvacuationBlock` in the range from *low* to *high*. Return 0 if no evacuation was started, non-zero otherwise.
+
+.. cpp:class:: CacheVol
+
+ A cache volume as described in :file:`volume.config`.
+
+.. cpp:class:: Doc
+
+ Defined in |P-CacheVol.h|_.
+
+ .. cpp:member:: uint32_t Doc::magic
+
+ Validity check value. Set to ``DOC_MAGIC`` for a valid document.
+
+ .. cpp:member:: uint32_t Doc::len
+
+ The length of this segment including the header length, fragment table, and this structure.
+
+ .. cpp:member:: uint64_t Doc::total_len
+
+ Total length of the entire document not including meta data but including headers.
+
+ .. cpp:member:: INK_MD5 Doc::first_key
+
+ First index key in the document (the index key used to locate this object in the volume index).
+
+ .. cpp:member:: INK_MD5 Doc::key
+
+ The index key for this fragment. Fragment keys are computationally chained so that the key for the next and previous fragments can be computed from this key.
+
+ .. cpp:member:: uint32_t Doc::hlen
+
+ Document header (metadata) length. This is not the length of the HTTP headers.
+
+ .. cpp:member:: uint8_t Doc::ftype
+
+ Fragment type. Currently only `CACHE_FRAG_TYPE_HTTP` is used. Other types may be used for cache extensions if those are ever used / implemented.
+
+ .. cpp:member:: uint24_t Doc::flen
+
+ Fragment table length, if any. Only the first ``Doc`` in an object should contain a fragment table.
+
+ The fragment table is a list of offsets relative to the HTTP content (not counting metadata or HTTP headers). Each offset is the byte offset of the first byte in the fragment. The first element in the table is the second fragment (what would be index 1 for an array). The offset for the first fragment is of course always zero and so not stored. The purpose of this is to enable a fast seek for range requests - given the first ``Doc`` the fragment containing the first byte in the range can be computed and loaded directly without further disk access.
+
+ Removed as of version 3.3.0.
+
+ .. cpp:member:: uint32_t Doc::sync_serial
+
+ Unknown.
+
+ .. cpp:member:: uint32_t Doc::write_serial
+
+ Unknown.
+
+ .. cpp:member:: uint32_t pinned
+
+ Flag and timer for pinned objects.
+
+ .. cpp:member:: uint32_t checksum
+
+ Unknown. (A checksum of some sort)
+
+.. rubric:: Footnotes
+
+.. [#] Changed in version 3.2.0. This previously resided in the first ``Doc`` but that caused different alternates to share the same fragment table.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/cache-glossary.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/cache-glossary.en.rst b/doc/arch/cache/cache-glossary.en.rst
new file mode 100755
index 0000000..6603895
--- /dev/null
+++ b/doc/arch/cache/cache-glossary.en.rst
@@ -0,0 +1,22 @@
+Glossary
+~~~~~~~~
+
+.. include:: common.defs
+
+.. glossary::
+ :sorted:
+
+ cache volume
+ Persistent storage for the cache, defined and manipulable by the user. Cache volumes are defined in :file:`volume.config`. A cache volume is spread across :term:`storage unit`\ s to increase performance through parallel I/O. Storage units can be split across cache volumes. Each such part of a storage unit in a cache volume is a :term:`volume`.
+
+ Implemented by the class :cpp:class:`CacheVol`.
+
+ volume
+ A homogenous persistent store for the cache. A volume always resides entirely on a single physical device and is treated as an undifferentiated span of bytes.
+
+ Implemented by the class :cpp:class:`Vol`.
+
+ See also :term:`storage unit`, :term:`cache volume`
+
+ storage unit
+ The physical storage described by a single line in :file:`storage.config`.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/cache.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/cache.en.rst b/doc/arch/cache/cache.en.rst
new file mode 100644
index 0000000..04f2cc2
--- /dev/null
+++ b/doc/arch/cache/cache.en.rst
@@ -0,0 +1,34 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Apache Traffic Server Cache
+***************************
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ cache-arch.en
+ cache-data-structures.en
+ cache-api.en
+ cache-appendix.en
+ cache-glossary.en
+ tier-storage.en
+
+.. glossary
+.. appendix
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/common.defs
----------------------------------------------------------------------
diff --git a/doc/arch/cache/common.defs b/doc/arch/cache/common.defs
new file mode 100644
index 0000000..7229c8a
--- /dev/null
+++ b/doc/arch/cache/common.defs
@@ -0,0 +1,31 @@
+.. Global definitions
+
+.. Files in github. Should write a custom domain so we can do :ts:git:`iocore/cache/P_CacheDir.h`
+
+.. |HTTP.h| replace:: ``HTTP.h``
+
+.. _HTTP.h: https://github.com/apache/trafficserver/blob/master/proxy/hdrs/HTTP.h
+
+.. |HttpCacheSM.h| replace:: ``HttpCacheSM.h``
+
+.. _HttpCacheSM.h: https://github.com/apache/trafficserver/blob/master/proxy/http/HttpCacheSM.h
+
+.. |HttpCacheSM.cc| replace:: ``HttpCacheSM.cc``
+
+.. _HttpCacheSM.cc: https://github.com/apache/trafficserver/blob/master/proxy/http/HttpCacheSM.cc
+
+.. |P-CacheDir.h| replace:: ``P_CacheDir.h``
+
+.. _P-CacheDir.h: https://github.com/apache/trafficserver/blob/master/iocore/cache/P_CacheDir.h
+
+.. |P-CacheHttp.h| replace:: ``P_CacheHttp.h``
+
+.. _P-CacheHttp.h: https://github.com/apache/trafficserver/blob/master/iocore/cache/P_CacheHttp.h
+
+.. |P-CacheVol.h| replace:: ``P_CacheVol.h``
+
+.. _P-CacheVol.h: https://github.com/apache/trafficserver/blob/master/iocore/cache/P_CacheVol.h
+
+.. |ATS| replace:: Apache Traffic Server
+
+.. |TS| replace:: Traffic Server
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-doc-layout-pre-3-2-0.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-doc-layout-pre-3-2-0.png b/doc/arch/cache/images/ats-cache-doc-layout-pre-3-2-0.png
new file mode 100755
index 0000000..d7b2c4f
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-doc-layout-pre-3-2-0.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-doc-layout.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-doc-layout.png b/doc/arch/cache/images/ats-cache-doc-layout.png
new file mode 100755
index 0000000..438ed77
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-doc-layout.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-layout.jpg
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-layout.jpg b/doc/arch/cache/images/ats-cache-layout.jpg
new file mode 100644
index 0000000..3c13487
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-layout.jpg differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-storage-units.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-storage-units.png b/doc/arch/cache/images/ats-cache-storage-units.png
new file mode 100644
index 0000000..47e1bda
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-storage-units.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-volume-definition.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-volume-definition.png b/doc/arch/cache/images/ats-cache-volume-definition.png
new file mode 100644
index 0000000..238d291
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-volume-definition.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-volume-directory.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-volume-directory.png b/doc/arch/cache/images/ats-cache-volume-directory.png
new file mode 100644
index 0000000..631ff9e
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-volume-directory.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-volume-layout.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-volume-layout.png b/doc/arch/cache/images/ats-cache-volume-layout.png
new file mode 100644
index 0000000..b05bf88
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-volume-layout.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/images/ats-cache-write-cursor.png
----------------------------------------------------------------------
diff --git a/doc/arch/cache/images/ats-cache-write-cursor.png b/doc/arch/cache/images/ats-cache-write-cursor.png
new file mode 100755
index 0000000..fe48679
Binary files /dev/null and b/doc/arch/cache/images/ats-cache-write-cursor.png differ
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/cache/tier-storage.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/cache/tier-storage.en.rst b/doc/arch/cache/tier-storage.en.rst
new file mode 100755
index 0000000..d7d5e81
--- /dev/null
+++ b/doc/arch/cache/tier-storage.en.rst
@@ -0,0 +1,57 @@
+==============================
+Tiered Storage Design
+==============================
+
+.. include:: common.defs
+
+--------------
+Introduction
+--------------
+
+Tiered storage is an attempt to allow |TS| to take advantage of physical storage with different properties. This design concerns only mechanism. Policies to take advantage of these are outside of the scope of this document. Instead we will presume an *oracle* which implements this policy and describe the queries that must be answered by the oracle and the effects of the answers.
+
+Beyond avoiding question of tier policy the design is also intended to be effectively identical to current operations for the case where there is only one tier.
+
+-------------
+Configuration
+-------------
+
+Each storage unit in :file:`storage.config` can be marked with a *quality* value which is 32 bit number. Storage units that are not marked are all assigned the same value which is guaranteed to be distinct from all explicit values.
+
+-------------
+Operations
+-------------
+
+When a request is received from a client volume assignment is done in parallel for each tier quality. The oracle is queried for each tier and returns a volume and one of four values
+
+`READ`
+ The tier appears to have the object and can serve it.
+
+`WRITE`
+ The object is not in this tier and should be written to this tier if possible.
+
+`RW`
+ Treat as `READ` if possible but if the object turns out to not in the cache treat as `WRITE`.
+
+`NO_SALE`
+ Do not interact with this tier for this object.
+
+The vvolume returned by the tier must be a volume with the corresponding quality value. In effect the current style of volume assignment is done for each tier, by assigning one volume out of all of the volumes of the same quality and returning one of `RW` or `WRITE` depending on whether the initial volume directory lookup succeeds.
+
+If the object is cached every tier that returns `WRITE` receives the object to store in the selected volume (this includes `RW` returns that are demoted to `WRITE`). If there is more than one `READ` or `RW` the oracle is consoluted (or configures at start time) an ordering of the tiers. The tiers are tried in order and the first that successfully accesses the object becomes the provider. If no tier is successful then the object is retrieved from the origin server.
+
+This means, among other things, that if there is a tier with the object all other tiers that are written will get a local copy of the object, the origin server will not be used. In terms of implementation, currently a cache write to a volume is done via the construction of an instance of `CacheVC` which recieves the object stream. For tiered storage the same thing is done for each target volume.
+
+For cache volume overrides (e.g. via :file:`hosting.config`) this same process is used except with only the volumes stripes contained within the specified cache volume.
+
+-------
+Copying
+-------
+
+It may be necessary to provide a mechanism to copy objects between tiers outside of a client originated transaction. In terms of implementation this is straight forward using `HttpTunnel` as if in a transaction only using a `CacheVC` instance for both the producer and consumer. The more difficult question is what event would trigger a possible copy. A signal could be provided whenever a volume directory entry is deleted although it should be noted that the object in question may have already been evicted when this event happens.
+
+----------------
+Additional Notes
+----------------
+
+As an example use, it would be possible to have only one cache volume that uses tiered storage for a particular set of domains using volume tagging. :file:`hosting.config` would be used to direct those domains to the selected cache volume. The oracle would check the URL in parallel and return `NO_SALE` for the tiers in the target cache volume for other domains. For the other tier (that of the unmarked storage units) the oracle would return `RW` for the tier in all cases as that tier would not be queried for the target domains.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7c21e68e/doc/arch/index.en.rst
----------------------------------------------------------------------
diff --git a/doc/arch/index.en.rst b/doc/arch/index.en.rst
new file mode 100644
index 0000000..4126dc1
--- /dev/null
+++ b/doc/arch/index.en.rst
@@ -0,0 +1,31 @@
+Architecture
+******************
+
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Introduction
+--------------
+
+The original architectural documents for Traffic Server were lost in the transation to an open source project. The documents in this section are provision and were written based on the existing code. The purpose is to have a high level description of aspects of Traffic Server to better inform ongoing work.
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ cache/cache.en