You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zy...@apache.org on 2012/11/07 17:06:12 UTC
git commit: TS-1561: Plugin esi - Enhancements on ESI plugin
Updated Branches:
refs/heads/master 53ad4f337 -> bed634b03
TS-1561: Plugin esi - Enhancements on ESI plugin
1) Runtime option to turn on packed node support
2) Runtime option to add headers "Expires: -1" and "Cache-Control:
max-age=0, private" to declare a content to be private
3) Updated README
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/bed634b0
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/bed634b0
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/bed634b0
Branch: refs/heads/master
Commit: bed634b03bc09c76ccfc9ace70039d688f6f71c7
Parents: 53ad4f3
Author: Kit Chan <ch...@gmail.com>
Authored: Wed Nov 7 23:44:55 2012 +0800
Committer: Zhao Yongming <mi...@gmail.com>
Committed: Thu Nov 8 00:02:37 2012 +0800
----------------------------------------------------------------------
CHANGES | 2 +
plugins/experimental/esi/Makefile.am | 1 -
plugins/experimental/esi/README | 137 +++++++--------
.../esi/fetcher/HttpDataFetcherImpl.cc | 12 +-
plugins/experimental/esi/lib/EsiProcessor.cc | 1 +
plugins/experimental/esi/plugin.cc | 109 +++++++-----
6 files changed, 145 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/bed634b0/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6e6eae5..a96f574 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,8 @@
-*- coding: utf-8 -*-
Changes with Apache Traffic Server 3.3.1
+ *) [TS-1561] Plugin esi - Enhancements on ESI plugin
+
*) [TS-1560] plugins need memory barriers for ARM
*) [TS-1558] use_client_addr breaks control over upstream HTTP protocol version
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/bed634b0/plugins/experimental/esi/Makefile.am
----------------------------------------------------------------------
diff --git a/plugins/experimental/esi/Makefile.am b/plugins/experimental/esi/Makefile.am
index 070c058..f9385fe 100644
--- a/plugins/experimental/esi/Makefile.am
+++ b/plugins/experimental/esi/Makefile.am
@@ -19,7 +19,6 @@ pkglibdir = ${pkglibexecdir}
AM_CXXFLAGS = \
-I$(srcdir)/lib \
- -DESI_PACKED_NODE_SUPPORT \
-I$(srcdir)/fetcher \
-I$(srcdir)/test \
-I$(top_builddir)/proxy/api \
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/bed634b0/plugins/experimental/esi/README
----------------------------------------------------------------------
diff --git a/plugins/experimental/esi/README b/plugins/experimental/esi/README
index 33f5ac1..7440ae5 100644
--- a/plugins/experimental/esi/README
+++ b/plugins/experimental/esi/README
@@ -30,74 +30,71 @@ Supported variables:
fg. such cookie string: l=u=test&t=1350952328, the value of $(HTTP_COOKIE{"l;u"}) is test
and the value of $(HTTP_COOKIE{"l;t"}) is 1350952328
-Version 1.3.0
--------------
-- Upgrading to yts_esi_lib-1.3.0 and using yts_http_fetcher_impl-1.0.0
-
-Version 1.2.0
--------------
-- Using INKVConnClose instead of INKVConnShutdown to avoid memory
- leak
-- Upgrading to yts_esi_lib-1.2.0
-
-Version 1.1.1
--------------
-- Suppressing non-200 status handling error message
-- Upgrading to yts_esi_lib-1.1.1
-
-Version 1.1.0
--------------
-- Suppressing non-200 status handling error message
-- Upgrading to yts_esi_lib-1.1.0
-
-Version 1.0.1
--------------
-- Added symlink from conf/yts/plugins for backward compatibility
-
-Version 1.0.0
--------------
-- Upgrading to yts_esi_lib-0.0.6
-
-Version 0.0.6
--------------
-- Removing X-Esi header from outbound client response
-
-Version 0.0.5_1
----------------
-- Requiring correct version of yts_esi_lib (0.0.5)
-
-Version 0.0.5
--------------
-- Using intercept solution to POST-back and cache serialized
- form of document
-- Added basic stats
-
-Version 0.0.4_2
----------------
-- Using yts_esi_lib-0.0.4
-
-Version 0.0.4_1
----------------
-- Added conf/yts_plugin_esi directory and handlers.cfg
-
-Version 0.0.4
--------------
-- Upgrading to yts_esi_lib{,_dev}-0.0.3
-- Special include handlers can now add 'footer' data
-- Dealing with gzip'ed data from OS and gzip'ing output if requested
-
-Version 0.0.3
--------------
-- Conforming to API changes becase of ticket 3515300
-- Partial fix to bug 3503119 (ESI plugin should return error code
- (4xx or 5xx) if it can not fetch include src url)
-
-
-Version 0.0.2
--------------
-- Upgraded to new fetch API, support for special includes
-
-Version 0.0.1
--------------
-- Prototype implementation
+Compile and Installation
+------------------------
+
+1) Just run "make" in the directory of trafficserver/plugins/experimental/esi/
+
+2) Copy trafficserver/plugins/experimental/esi/.libs/esi.* to /usr/local/libexec/trafficserver/
+
+3) Copy trafficserver/plugins/experimental/esi/.libs/libesi.* to /usr/local/lib/
+
+Enabling ESI
+------------
+
+1) First we need to set up /usr/local/etc/trafficserver/plugin.config and make sure the following line is present
+
+esi.so
+
+There are two options you can add. "--private_response" will add private cache control and expires header to the processed ESI document. "--packed_node_support" will enable the support for using packed node, which will improve the performance of parsing cached ESI document.
+
+2) We need a mapping for origin server response that contains the ESI markup. Assume that the ATS server is abc.com. And your origin server is xyz.com and the response containing ESI markup is http://xyz.com/esi.php. We will need the following line in /usr/local/etc/trafficserver/remap.config
+
+map http://abc.com/esi.php http://xyz.com/esi.php
+
+3) Your response should contain ESI markup and a response header of .X-Esi: 1'. e.g. using PHP,
+
+<?php header('X-Esi: 1'); ?>
+<html>
+<body>
+Hello, <esi:include src="http://abc.com/date.php"/>
+</body>
+</html>
+
+4) You will need a mapping for the src of the ESI include in remap.config if it is not already present.
+
+map http://abc.com/date.php http://xyz.com/date.php
+
+Or if both your ESI response and the ESI include comes from the same origin server, you can have the following line in remap.config instead to replace separate map rules for date.php and esi.php
+
+map http://abc.com/ http://xyz.com/
+
+5) Here is a sample PHP for date.php
+
+<?php
+ header ("Cache-control: no-cache");
+ echo date('l jS \of F Y h:i:s A');
+?>
+
+Useful Note
+-----------
+
+1) You can provide proper cache control header and the ESI response and ESI include response can be cached separately. It is extremely useful for rendering page with multiple modules. The page layout can be a ESI response with multiple ESI include, each for different module. The page layour ESI response can be cached and each individual ESI include can also be cached with different duration.
+
+2) You might want to compile the code without using ESI_PACKED_NODE_SUPPORT because it may not work in some corner cases
+
+Differences from Spec - http://www.w3.org/TR/esi-lang
+-----------------------------------------------------
+
+1) <esi:include> does not support "alt" and "onerror" attributes
+
+2) <esi:inline> is not supported
+
+3) You cannot have <esi:try> inside another <esi:try>
+
+4) HTTP_USER_AGENT variable is not supported
+
+5) <!--esi .. --> is not supported
+
+6) HTTP_COOKIE supports fetching for sub-key
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/bed634b0/plugins/experimental/esi/fetcher/HttpDataFetcherImpl.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/esi/fetcher/HttpDataFetcherImpl.cc b/plugins/experimental/esi/fetcher/HttpDataFetcherImpl.cc
index b89ba28..aeb8711 100644
--- a/plugins/experimental/esi/fetcher/HttpDataFetcherImpl.cc
+++ b/plugins/experimental/esi/fetcher/HttpDataFetcherImpl.cc
@@ -166,10 +166,6 @@ HttpDataFetcherImpl::handleFetchEvent(TSEvent event, void *edata) {
TSDebug(_debug_tag,
"[%s] Inserted page data of size %d starting with [%.6s] for request [%s]", __FUNCTION__,
req_data.body_len, (req_data.body_len ? req_data.body : "(null)"), req_str.c_str());
- for (CallbackObjectList::iterator list_iter = req_data.callback_objects.begin();
- list_iter != req_data.callback_objects.end(); ++list_iter) {
- (*list_iter)->processData(req_str.data(), req_str.size(), req_data.body, req_data.body_len);
- }
if (_checkHeaderValue(req_data.bufp, req_data.hdr_loc,
TS_MIME_FIELD_CONTENT_ENCODING,
@@ -187,6 +183,12 @@ HttpDataFetcherImpl::handleFetchEvent(TSEvent event, void *edata) {
TSError("[%s] Error while gunzipping data", __FUNCTION__);
}
}
+
+ for (CallbackObjectList::iterator list_iter = req_data.callback_objects.begin();
+ list_iter != req_data.callback_objects.end(); ++list_iter) {
+ (*list_iter)->processData(req_str.data(), req_str.size(), req_data.body, req_data.body_len);
+ }
+
} else {
TSDebug(_debug_tag, "[%s] Received non-OK status %d for request [%s]",
__FUNCTION__, resp_status, req_str.data());
@@ -298,11 +300,13 @@ HttpDataFetcherImpl::getRequestStatus(const string &url) const {
void
HttpDataFetcherImpl::useHeader(const HttpHeader &header) {
+ // request data body would not be passed to async request and so we should not pass on the content length
if (Utils::areEqual(header.name, header.name_len,
TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH)) {
return;
}
+ // should not support partial request for async request
if (Utils::areEqual(header.name, header.name_len,
TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE)) {
return;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/bed634b0/plugins/experimental/esi/lib/EsiProcessor.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/esi/lib/EsiProcessor.cc b/plugins/experimental/esi/lib/EsiProcessor.cc
index 4a6b10b..48677b4 100644
--- a/plugins/experimental/esi/lib/EsiProcessor.cc
+++ b/plugins/experimental/esi/lib/EsiProcessor.cc
@@ -229,6 +229,7 @@ EsiProcessor::process(const char *&data, int &data_len) {
attemptUrls.push_back(_expression.expand(raw_url));
if (!_getIncludeData(*node_iter)) {
attempt_succeeded = false;
+ _errorLog("[%s] attempt section errored; due to url [%s]", __FUNCTION__, raw_url.c_str());
break;
}
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/bed634b0/plugins/experimental/esi/plugin.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/esi/plugin.cc b/plugins/experimental/esi/plugin.cc
index a2b74f5..37ea67a 100644
--- a/plugins/experimental/esi/plugin.cc
+++ b/plugins/experimental/esi/plugin.cc
@@ -33,6 +33,7 @@
#include <list>
#include <arpa/inet.h>
#include <pthread.h>
+#include <getopt.h>
#include "ts/ts.h"
#include "ts/experimental.h"
@@ -50,6 +51,11 @@ using std::list;
using namespace EsiLib;
using namespace Stats;
+struct OptionInfo
+{
+ bool packed_node_support, private_response;
+} option_info = {false, false};
+
static HandlerManager *gHandlerManager;
#define DEBUG_TAG "plugin_esi"
@@ -63,6 +69,9 @@ static HandlerManager *gHandlerManager;
#define MIME_FIELD_XESI "X-Esi"
#define MIME_FIELD_XESI_LEN 5
+static const char HTTP_VALUE_PRIVATE_EXPIRES[] = "-1";
+static const char HTTP_VALUE_PRIVATE_CC[] = "max-age=0, private";
+
enum DataType { DATA_TYPE_RAW_ESI = 0, DATA_TYPE_GZIPPED_ESI = 1, DATA_TYPE_PACKED_ESI = 2 };
static const char *DATA_TYPE_NAMES_[] = { "RAW_ESI",
"GZIPPED_ESI",
@@ -97,10 +106,8 @@ struct ContData
bool cache_txn;
bool head_only;
-#ifdef ESI_PACKED_NODE_SUPPORT
bool os_response_cacheable;
list<string> post_headers;
-#endif
ContData(TSCont contptr, TSHttpTxn tx)
: curr_state(READING_ESI_DOC), input_vio(NULL),
@@ -111,9 +118,7 @@ struct ContData
gzipped_data(""), gzip_output(false),
initialized(false), xform_closed(false),
intercept_header(false), cache_txn(false), head_only(false)
-#ifdef ESI_PACKED_NODE_SUPPORT
, os_response_cacheable(true)
-#endif
{
client_addr = TSHttpTxnClientAddrGet(txnp);
*debug_tag = '\0';
@@ -326,7 +331,6 @@ ContData::getClientState() {
TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_hdr_loc);
}
-#ifdef ESI_PACKED_NODE_SUPPORT
void
ContData::fillPostHeader(TSMBuffer bufp, TSMLoc hdr_loc) {
int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc);
@@ -395,7 +399,6 @@ ContData::fillPostHeader(TSMBuffer bufp, TSMLoc hdr_loc) {
}
} // end header iteration
}
-#endif
void
ContData::getServerState() {
@@ -424,11 +427,9 @@ ContData::getServerState() {
input_type = DATA_TYPE_RAW_ESI;
}
-#ifdef ESI_PACKED_NODE_SUPPORT
- if (!cache_txn && !head_only) {
+ if ( option_info.packed_node_support && !cache_txn && !head_only) {
fillPostHeader(bufp, hdr_loc);
}
-#endif
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
}
@@ -521,7 +522,6 @@ static bool removeCacheKey(TSHttpTxn txnp) {
return result;
}
-#ifdef ESI_PACKED_NODE_SUPPORT
static void
cacheNodeList(ContData *cont_data) {
if (TSHttpTxnAborted(cont_data->txnp) == TS_SUCCESS) {
@@ -567,7 +567,6 @@ cacheNodeList(ContData *cont_data) {
TSFetchUrl(post_request.data(), post_request.size(), cont_data->client_addr,
cont_data->contp, NO_CALLBACK, event_ids);
}
-#endif
static int
transformData(TSCont contp)
@@ -684,11 +683,9 @@ transformData(TSCont contp)
}
}
if (cont_data->esi_proc->completeParse()) {
-#ifdef ESI_PACKED_NODE_SUPPORT
- if (cont_data->os_response_cacheable && !cont_data->cache_txn && !cont_data->head_only) {
+ if ( option_info.packed_node_support && cont_data->os_response_cacheable && !cont_data->cache_txn && !cont_data->head_only) {
cacheNodeList(cont_data);
}
-#endif
}
}
@@ -967,11 +964,11 @@ modifyResponseHeader(TSCont contp, TSEvent event, void *edata) {
destroy_header = true;
} else if ((name_len > HEADER_MASK_PREFIX_SIZE) &&
(strncmp(name, HEADER_MASK_PREFIX, HEADER_MASK_PREFIX_SIZE) == 0))
- {
+ {
+ destroy_header = true;
+ } else if (option_info.private_response && (Utils::areEqual(name, name_len, TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL) || Utils::areEqual(name, name_len, TS_MIME_FIELD_EXPIRES, TS_MIME_LEN_EXPIRES))) {
destroy_header = true;
- } else if (Utils::areEqual(name, name_len,
- TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH))
- {
+ } else if (Utils::areEqual(name, name_len, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH)) {
have_content_length = true;
if (mod_data->head_only) {
destroy_header = true;
@@ -985,17 +982,13 @@ modifyResponseHeader(TSCont contp, TSEvent event, void *edata) {
TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]",
__FUNCTION__, j, name_len, name);
} else {
-#ifdef ESI_PACKED_NODE_SUPPORT
- if (mod_data->cache_txn) {
-#endif
+ if (!option_info.packed_node_support || mod_data->cache_txn) {
bool response_cacheable, is_cache_header;
is_cache_header = checkForCacheHeader(name, name_len, value, value_len, response_cacheable);
if (is_cache_header && response_cacheable) {
destroy_header = true;
}
-#ifdef ESI_PACKED_NODE_SUPPORT
}
-#endif
} // if got valid value for header
} // end for
}
@@ -1019,12 +1012,16 @@ modifyResponseHeader(TSCont contp, TSEvent event, void *edata) {
TSError("[%s] no Content-Length!", __FUNCTION__);
}
-#ifdef ESI_PACKED_NODE_SUPPORT
- if (mod_data->cache_txn) {
+ if (option_info.packed_node_support && mod_data->cache_txn) {
addMimeHeaderField(bufp, hdr_loc, TS_MIME_FIELD_VARY, TS_MIME_LEN_VARY, TS_MIME_FIELD_ACCEPT_ENCODING,
TS_MIME_LEN_ACCEPT_ENCODING);
}
-#endif
+
+ if(option_info.private_response) {
+ addMimeHeaderField(bufp, hdr_loc, TS_MIME_FIELD_EXPIRES, TS_MIME_LEN_EXPIRES, HTTP_VALUE_PRIVATE_EXPIRES, strlen(HTTP_VALUE_PRIVATE_EXPIRES));
+ addMimeHeaderField(bufp, hdr_loc, TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL, HTTP_VALUE_PRIVATE_CC, strlen(HTTP_VALUE_PRIVATE_CC));
+ }
+
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
TSDebug(DEBUG_TAG, "[%s] Inspected client-bound headers", __FUNCTION__);
retval = 1;
@@ -1077,7 +1074,6 @@ checkHeaderValue(TSMBuffer bufp, TSMLoc hdr_loc, const char *name, int name_len,
return retval;
}
-#ifdef ESI_PACKED_NODE_SUPPORT
static void
maskOsCacheHeaders(TSHttpTxn txnp) {
TSMBuffer bufp;
@@ -1136,7 +1132,6 @@ maskOsCacheHeaders(TSHttpTxn txnp) {
} // end header iteration
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
}
-#endif
static bool
isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool * intercept_header, bool * head_only) {
@@ -1290,6 +1285,8 @@ checkForCacheHeader(const char *name, int name_len, const char *value, int value
if (Utils::areEqual(name, name_len, TS_MIME_FIELD_EXPIRES, TS_MIME_LEN_EXPIRES)) {
if ((value_len == 1) && (*value == '0')) {
cacheable = false;
+ }else if (Utils::areEqual(value, value_len, "-1",2)) {
+ cacheable = false;
}
return true;
}
@@ -1339,15 +1336,15 @@ addTransform(TSHttpTxn txnp, const bool processing_os_response, const bool inter
cont_data->getServerState();
if (cont_data->cache_txn) {
-#ifdef ESI_PACKED_NODE_SUPPORT
+ if(option_info.packed_node_support) {
if (cont_data->input_type != DATA_TYPE_PACKED_ESI) {
removeCacheKey(txnp);
}
-#else
+ } else {
if (cont_data->input_type == DATA_TYPE_PACKED_ESI) {
removeCacheKey(txnp);
}
-#endif
+ }
}
TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, contp);
@@ -1358,11 +1355,11 @@ addTransform(TSHttpTxn txnp, const bool processing_os_response, const bool inter
}
TSHttpTxnTransformedRespCache(txnp, 0);
-#ifdef ESI_PACKED_NODE_SUPPORT
- TSHttpTxnUntransformedRespCache(txnp, 0);
-#else
- TSHttpTxnUntransformedRespCache(txnp, 1);
-#endif
+ if(option_info.packed_node_support) {
+ TSHttpTxnUntransformedRespCache(txnp, 0);
+ } else {
+ TSHttpTxnUntransformedRespCache(txnp, 1);
+ }
TSDebug(DEBUG_TAG, "[%s] Added transformation (0x%p)", __FUNCTION__, contp);
return true;
@@ -1417,13 +1414,11 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata) {
Stats::increment(Stats::N_OS_DOCS);
mask_cache_headers = true;
}
- if (mask_cache_headers) {
+ if (option_info.packed_node_support && mask_cache_headers) {
// we'll 'mask' OS cache headers so that traffic server will
// not try to cache this. We cannot outright delete them
// because we need them in our POST request; hence the 'masking'
-#ifdef ESI_PACKED_NODE_SUPPORT
maskOsCacheHeaders(txnp);
-#endif
}
} else {
TSDebug(DEBUG_TAG, "[%s] handling cache lookup complete event...", __FUNCTION__);
@@ -1472,10 +1467,40 @@ TSPluginInit(int argc, const char *argv[]) {
gHandlerManager = new HandlerManager(HANDLER_MGR_DEBUG_TAG, &TSDebug, &TSError);
- if ((argc > 1) && (strcmp(argv[1], "-") != 0)) {
+ //if ((argc > 1) && (strcmp(argv[1], "-") != 0)) {
+ // Utils::KeyValueMap handler_conf;
+ // loadHandlerConf(argv[1], handler_conf);
+ // gHandlerManager->loadObjects(handler_conf);
+ //}
+ if (argc > 1)
+ {
+ int c;
Utils::KeyValueMap handler_conf;
- loadHandlerConf(argv[1], handler_conf);
- gHandlerManager->loadObjects(handler_conf);
+ static const struct option longopts[] = {
+ { "packed-node-support", no_argument, NULL, 'n' },
+ { "private-response", no_argument, NULL, 'p' },
+ { "handler-filename", required_argument, NULL, 'f' },
+ { NULL, 0, NULL, 0 }
+ };
+
+ while ((c = getopt_long(argc, (char * const*) argv, "npf:", longopts, NULL)) != -1)
+ {
+ switch (c)
+ {
+ case 'n':
+ option_info.packed_node_support = true;
+ break;
+ case 'p':
+ option_info.private_response = true;
+ break;
+ case 'f':
+ loadHandlerConf(optarg, handler_conf);
+ gHandlerManager->loadObjects(handler_conf);
+ break;
+ default:
+ break;
+ }
+ }
}
if(pthread_key_create(&threadKey,NULL)){