You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by os...@apache.org on 2014/08/04 15:57:48 UTC
[1/3] ats_speed: PageSpeed optimization plugin
Repository: trafficserver
Updated Branches:
refs/heads/master fab202517 -> 083abd4ff
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/README
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/README b/plugins/experimental/ats_speed/gzip/README
new file mode 100644
index 0000000..2e74681
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/README
@@ -0,0 +1,4 @@
+This gzip is not compiled, but only here to be able to diff later
+with the official gzip plugin. It is very slightly modified no
+be able to disable caching of its compressed output through a
+response header
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/configuration.cc b/plugins/experimental/ats_speed/gzip/configuration.cc
new file mode 100644
index 0000000..b1c499d
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/configuration.cc
@@ -0,0 +1,264 @@
+/** @file
+
+ Transforms content using gzip or deflate
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "configuration.h"
+#include <fstream>
+#include <algorithm>
+#include <vector>
+#include <fnmatch.h>
+
+namespace Gzip {
+ using namespace std;
+
+ void ltrim_if(string& s, int (* fp) (int)) {
+ for (size_t i = 0; i < s.size();) {
+ if (fp(s[i])) {
+ s.erase(i,1);
+ } else {
+ break;
+ }
+ }
+ }
+
+ void rtrim_if(string& s, int (* fp) (int)) {
+ for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) {
+ if (fp(s[i])) {
+ s.erase(i,1);
+ } else {
+ break;
+ }
+ }
+ }
+
+ void trim_if(string& s, int (* fp) (int)) {
+ ltrim_if(s, fp);
+ rtrim_if(s, fp);
+ }
+
+ vector<string> tokenize(const string &s, int (* fp) (int)) {
+ vector<string> r;
+ string tmp;
+
+ for (size_t i = 0; i < s.size(); i++) {
+ if ( fp(s[i]) ) {
+ if ( tmp.size() ) {
+ r.push_back(tmp);
+ tmp = "";
+ }
+ } else {
+ tmp += s[i];
+ }
+ }
+
+ if ( tmp.size() ) {
+ r.push_back(tmp);
+ }
+
+ return r;
+ }
+
+ enum ParserState {
+ kParseStart,
+ kParseCompressibleContentType,
+ kParseRemoveAcceptEncoding,
+ kParseEnable,
+ kParseCache,
+ kParseDisallow,
+ };
+
+ void Configuration::AddHostConfiguration(HostConfiguration * hc){
+ host_configurations_.push_back(hc);
+ }
+
+ void HostConfiguration::add_disallow(const std::string & disallow) {
+ disallows_.push_back(disallow);
+ }
+
+ void HostConfiguration::add_compressible_content_type(const std::string & content_type) {
+ compressible_content_types_.push_back(content_type);
+ }
+
+ HostConfiguration * Configuration::Find(const char * host, int host_length) {
+ HostConfiguration * host_configuration = host_configurations_[0];
+
+ std::string shost(host, host_length);
+
+ for (size_t i = 1; i < host_configurations_.size(); i++ ) {
+ if (host_configurations_[i]->host() == shost){
+ host_configuration = host_configurations_[i];
+ break;
+ }
+ }
+
+ return host_configuration;
+ }
+
+ bool HostConfiguration::IsUrlAllowed(const char * url, int url_len) {
+ string surl(url, url_len);
+
+ for (size_t i = 0; i < disallows_.size(); i++) {
+ if ( fnmatch (disallows_[i].c_str(), surl.c_str(), 0) == 0 ) {
+ info("url [%s] disabled for compression, matched on pattern [%s]",
+ surl.c_str(), disallows_[i].c_str());
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ bool HostConfiguration::ContentTypeIsCompressible(const char * content_type, int content_type_length) {
+ string scontent_type(content_type, content_type_length);
+ bool is_match = false;
+
+ for (size_t i = 0; i < compressible_content_types_.size(); i++) {
+ const char* match_string = compressible_content_types_[i].c_str();
+ bool exclude = match_string[0] == '!';
+ if (exclude) {
+ match_string++;//skip '!'
+ }
+ if ( fnmatch (match_string, scontent_type.c_str(), 0) == 0 ) {
+ info("compressible content type [%s], matched on pattern [%s]",
+ scontent_type.c_str(), compressible_content_types_[i].c_str());
+ is_match = !exclude;
+ }
+ }
+
+ return is_match;
+ }
+
+ Configuration * Configuration::Parse(const char * path ) {
+ string pathstring(path);
+
+ // If we have a path and it's not an absolute path, make it relative to the
+ // configuration directory.
+ if (!pathstring.empty() && pathstring[0] != '/') {
+ pathstring.assign(TSConfigDirGet());
+ pathstring.append("/");
+ pathstring.append(path);
+ }
+
+ trim_if(pathstring, isspace);
+
+ Configuration * c = new Configuration();
+ HostConfiguration * current_host_configuration = new HostConfiguration("");
+ c->AddHostConfiguration(current_host_configuration);
+ current_host_configuration->add_compressible_content_type("text/*");
+ current_host_configuration->add_compressible_content_type("application/xml*");
+ current_host_configuration->add_compressible_content_type("*javascript*");
+ current_host_configuration->add_compressible_content_type("image/svg+xml");
+
+
+ if (pathstring.empty()) {
+ return c;
+ }
+
+ path = pathstring.c_str();
+ info("Parsing file \"%s\"", path);
+ std::ifstream f;
+
+ size_t lineno = 0;
+
+ f.open(path, std::ios::in);
+
+ if (!f.is_open()) {
+ warning("could not open file [%s], skip",path);
+ return c;
+ }
+
+ enum ParserState state = kParseStart;
+
+ while (!f.eof()) {
+ std::string line;
+ getline(f, line);
+ ++lineno;
+
+ trim_if(line, isspace);
+ if (line.size() == 0) {
+ continue;
+ }
+
+ vector<string> v = tokenize( line, isspace );
+
+ for(size_t i = 0; i < v.size(); i++ ) {
+ string token = v[i];
+ trim_if(token, isspace);
+
+ //should not happen
+ if (!token.size()) continue;
+
+ //once a comment is encountered, we are done processing the line
+ if (token[0] == '#') break;
+
+ switch(state) {
+ case kParseStart:
+ if ( (token[0] == '[') && (token[token.size()-1] == ']')){
+ std::string current_host = token.substr(1,token.size()-2);
+ current_host_configuration = new HostConfiguration(current_host);
+ c->AddHostConfiguration(current_host_configuration);
+ } else if (token == "compressible-content-type" ) {
+ state = kParseCompressibleContentType;
+ } else if (token == "remove-accept-encoding" ) {
+ state = kParseRemoveAcceptEncoding;
+ } else if (token == "enabled" ) {
+ state = kParseEnable;
+ } else if (token == "cache" ) {
+ state = kParseCache;
+ } else if (token == "disallow" ) {
+ state = kParseDisallow;
+ }
+ else {
+ warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno);
+ }
+ break;
+ case kParseCompressibleContentType:
+ current_host_configuration->add_compressible_content_type(token);
+ state = kParseStart;
+ break;
+ case kParseRemoveAcceptEncoding:
+ current_host_configuration->set_remove_accept_encoding(token == "true");
+ state = kParseStart;
+ break;
+ case kParseEnable:
+ current_host_configuration->set_enabled(token == "true");
+ state = kParseStart;
+ break;
+ case kParseCache:
+ current_host_configuration->set_cache(token == "true");
+ state = kParseStart;
+ break;
+ case kParseDisallow:
+ current_host_configuration->add_disallow(token);
+ state = kParseStart;
+ break;
+ }
+ }
+ }
+
+ if (state != kParseStart) {
+ warning("the parser state indicates that data was expected when it reached the end of the file (%d)", state);
+ }
+
+ return c;
+ } //Configuration::Parse
+} //namespace
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/configuration.h b/plugins/experimental/ats_speed/gzip/configuration.h
new file mode 100644
index 0000000..b38cb64
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/configuration.h
@@ -0,0 +1,84 @@
+/** @file
+
+ Transforms content using gzip or deflate
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+
+#ifndef GZIP_CONFIGURATION_H_
+#define GZIP_CONFIGURATION_H_
+
+#include <string>
+#include <vector>
+#include "debug_macros.h"
+
+namespace Gzip {
+ class HostConfiguration {
+ public: //todo -> only configuration should be able to construct hostconfig
+ explicit HostConfiguration(const std::string & host)
+ : host_(host)
+ , enabled_(true)
+ , cache_(true)
+ , remove_accept_encoding_(false)
+ {}
+
+ inline bool enabled() { return enabled_; }
+ inline void set_enabled(bool x) { enabled_ = x; }
+ inline bool cache() { return cache_; }
+ inline void set_cache(bool x) { cache_ = x; }
+ inline bool remove_accept_encoding() { return remove_accept_encoding_; }
+ inline void set_remove_accept_encoding(bool x) { remove_accept_encoding_ = x; }
+ inline std::string host() { return host_; }
+ void add_disallow(const std::string & disallow);
+ void add_compressible_content_type(const std::string & content_type);
+ bool IsUrlAllowed(const char * url, int url_len);
+ bool ContentTypeIsCompressible(const char * content_type, int content_type_length);
+
+ private:
+ std::string host_;
+ bool enabled_;
+ bool cache_;
+ bool remove_accept_encoding_;
+ std::vector<std::string> compressible_content_types_;
+ std::vector<std::string> disallows_;
+ DISALLOW_COPY_AND_ASSIGN(HostConfiguration);
+ };//class HostConfiguration
+
+ class Configuration {
+ friend class HostConfiguration;
+ public:
+ static Configuration * Parse(const char * path);
+ HostConfiguration * Find(const char * host, int host_length);
+ inline HostConfiguration * GlobalConfiguration() {
+ return host_configurations_[0];
+ }
+
+ private:
+ explicit Configuration() {}
+ void AddHostConfiguration(HostConfiguration * hc);
+
+ std::vector<HostConfiguration *> host_configurations_;
+ //todo: destructor. delete owned host configurations
+ DISALLOW_COPY_AND_ASSIGN(Configuration);
+ }; //class Configuration
+
+}//namespace
+
+#endif
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/debug_macros.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/debug_macros.h b/plugins/experimental/ats_speed/gzip/debug_macros.h
new file mode 100644
index 0000000..151de31
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/debug_macros.h
@@ -0,0 +1,59 @@
+/** @file
+
+ Transforms content using gzip or deflate
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#ifndef _DBG_MACROS_H
+#define _DBG_MACROS_H
+
+#include <ts/ts.h>
+
+#define TAG "gzip"
+
+#define debug(fmt, args...) do { \
+ TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ } while (0)
+
+#define info(fmt, args...) do { \
+ TSDebug(TAG, "INFO: " fmt, ##args ); \
+ } while (0)
+
+#define warning(fmt, args...) do { \
+ TSDebug(TAG, "WARNING: " fmt, ##args ); \
+} while (0)
+
+#define error(fmt, args...) do { \
+ TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+} while (0)
+
+#define fatal(fmt, args...) do { \
+ TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ exit(-1); \
+} while (0)
+
+//FIXME: this one doesn't deserve to be here
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+ TypeName(const TypeName&); \
+ void operator=(const TypeName&)
+
+#endif //_DBG_MACROS_H
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/gzip.cc b/plugins/experimental/ats_speed/gzip/gzip.cc
new file mode 100644
index 0000000..1397762
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/gzip.cc
@@ -0,0 +1,826 @@
+/** @file
+
+ Transforms content using gzip or deflate
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+#define __STDC_LIMIT_MACROS
+#define __STDC_CONSTANT_MACROS
+#include <stdint.h>
+
+#include <string>
+#include <string.h>
+#include <zlib.h>
+#include <ts/ts.h>
+#include "debug_macros.h"
+#include "misc.h"
+#include "configuration.h"
+//#include "ink_defs.h"
+
+using namespace std;
+using namespace Gzip;
+
+//FIXME: custom dictionaries would be nice. configurable/content-type?
+//FIXME: look into autoscaling the compression level based on connection speed
+// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
+// might be served better with little or no compression at all
+//FIXME: look into compressing from the task thread pool
+//FIXME: make normalizing accept encoding configurable
+
+// from mod_deflate:
+// ZLIB's compression algorithm uses a
+// 0-9 based scale that GZIP does where '1' is 'Best speed'
+// and '9' is 'Best compression'. Testing has proved level '6'
+// to be about the best level to use in an HTTP Server.
+
+const int ZLIB_COMPRESSION_LEVEL = 6;
+
+int arg_idx_hooked;
+int arg_idx_host_configuration;
+int arg_idx_url_disallowed;
+
+
+const char * global_hidden_header_name;
+Configuration* config = NULL;
+const char *dictionary = NULL;
+
+static GzipData *
+gzip_data_alloc(int compression_type)
+{
+ GzipData *data;
+ int err;
+
+ data = (GzipData *) TSmalloc(sizeof(GzipData));
+ data->downstream_vio = NULL;
+ data->downstream_buffer = NULL;
+ data->downstream_reader = NULL;
+ data->downstream_length = 0;
+ data->state = transform_state_initialized;
+ data->compression_type = compression_type;
+ data->zstrm.next_in = Z_NULL;
+ data->zstrm.avail_in = 0;
+ data->zstrm.total_in = 0;
+ data->zstrm.next_out = Z_NULL;
+ data->zstrm.avail_out = 0;
+ data->zstrm.total_out = 0;
+ data->zstrm.zalloc = gzip_alloc;
+ data->zstrm.zfree = gzip_free;
+ data->zstrm.opaque = (voidpf) 0;
+ data->zstrm.data_type = Z_ASCII;
+
+ int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
+
+ err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
+
+ if (err != Z_OK) {
+ fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
+ }
+
+ if (dictionary) {
+ err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
+ if (err != Z_OK) {
+ fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
+ }
+ }
+
+ return data;
+}
+
+
+static void
+gzip_data_destroy(GzipData * data)
+{
+ TSReleaseAssert(data);
+
+ //deflateEnd returnvalue ignore is intentional
+ //it would spew log on every client abort
+ deflateEnd(&data->zstrm);
+
+ if (data->downstream_buffer) {
+ TSIOBufferDestroy(data->downstream_buffer);
+ }
+
+ TSfree(data);
+}
+
+static TSReturnCode
+gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type)
+{
+ TSReturnCode ret;
+ TSMLoc ce_loc;
+
+ // Delete Content-Encoding if present???
+
+ if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Content-Encoding", sizeof("Content-Encoding") - 1, &ce_loc)) == TS_SUCCESS) {
+ if (compression_type == COMPRESSION_TYPE_DEFLATE) {
+ ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", sizeof("deflate") - 1);
+ } else if (compression_type == COMPRESSION_TYPE_GZIP) {
+ ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", sizeof("gzip") - 1);
+ }
+ if (ret == TS_SUCCESS) {
+ ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+ }
+
+ if (ret != TS_SUCCESS) {
+ error("cannot add the Content-Encoding header");
+ }
+
+ return ret;
+}
+
+static TSReturnCode
+gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
+{
+ TSReturnCode ret;
+ TSMLoc ce_loc;
+
+ ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Vary", sizeof("Vary") - 1);
+ if (ce_loc) {
+ int idx, count, len;
+ const char *value;
+
+ count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc);
+ for(idx=0; idx<count; idx++) {
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len);
+ if (len &&
+ strncasecmp("Accept-Encoding", value, len) == 0) {
+ // Bail, Vary: Accept-Encoding already sent from origin
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+ return TS_SUCCESS;
+ }
+ }
+
+ ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1);
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+ } else {
+ if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Vary", sizeof("Vary") - 1, &ce_loc)) == TS_SUCCESS) {
+ if ((ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1)) == TS_SUCCESS) {
+ ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
+ }
+
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+ }
+ }
+
+ if (ret != TS_SUCCESS) {
+ error("cannot add/update the Vary header");
+ }
+
+ return ret;
+}
+
+//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
+// specify a very header..
+static TSReturnCode
+gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
+{
+ TSReturnCode ret = TS_SUCCESS;
+ TSMLoc ce_loc;
+
+ ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
+
+ if (ce_loc) {
+ int changetag = 1;
+ int strl;
+ const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
+ //do not alter weak etags.
+ //FIXME: consider just making the etag weak for compressed content
+ if (strl >= 2) {
+ if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
+ changetag = 0;
+ }
+ if (changetag) {
+ ret = TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
+ }
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+ }
+
+ if (ret != TS_SUCCESS) {
+ error("cannot handle the %s header", TS_MIME_FIELD_ETAG);
+ }
+
+ return ret;
+}
+
+//FIXME: some things are potentially compressible. those responses
+static void
+gzip_transform_init(TSCont contp, GzipData * data)
+{
+ //update the vary, content-encoding, and etag response headers
+ //prepare the downstream for transforming
+
+ TSVConn downstream_conn;
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+
+ data->state = transform_state_output;
+
+ if (TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc) != TS_SUCCESS) {
+ error("Error TSHttpTxnTransformRespGet");
+ return;
+ }
+
+ if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == TS_SUCCESS &&
+ gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS &&
+ gzip_etag_header(bufp, hdr_loc) == TS_SUCCESS) {
+ downstream_conn = TSTransformOutputVConnGet(contp);
+ data->downstream_buffer = TSIOBufferCreate();
+ data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
+ data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
+ }
+
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+}
+
+
+
+static void
+gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
+{
+ TSIOBufferBlock downstream_blkp;
+ const char *upstream_buffer;
+ char *downstream_buffer;
+ int64_t upstream_length, downstream_length;
+ int err;
+
+ while (amount > 0) {
+ downstream_blkp = TSIOBufferReaderStart(upstream_reader);
+ if (!downstream_blkp) {
+ error("couldn't get from IOBufferBlock");
+ return;
+ }
+
+ upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
+ if (!upstream_buffer) {
+ error("couldn't get from TSIOBufferBlockReadStart");
+ return;
+ }
+
+ if (upstream_length > amount) {
+ upstream_length = amount;
+ }
+
+ data->zstrm.next_in = (unsigned char *) upstream_buffer;
+ data->zstrm.avail_in = upstream_length;
+
+ while (data->zstrm.avail_in > 0) {
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+
+ data->zstrm.next_out = (unsigned char *) downstream_buffer;
+ data->zstrm.avail_out = downstream_length;
+
+ err = deflate(&data->zstrm, Z_NO_FLUSH);
+
+ if (err != Z_OK)
+ warning("deflate() call failed: %d", err);
+
+ if (downstream_length > data->zstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
+ data->downstream_length += (downstream_length - data->zstrm.avail_out);
+ }
+
+ if (data->zstrm.avail_out > 0) {
+ if (data->zstrm.avail_in != 0) {
+ error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
+ }
+ }
+ }
+
+ TSIOBufferReaderConsume(upstream_reader, upstream_length);
+ amount -= upstream_length;
+ }
+}
+
+static void
+gzip_transform_finish(GzipData * data)
+{
+ if (data->state == transform_state_output) {
+ TSIOBufferBlock downstream_blkp;
+ char *downstream_buffer;
+ int64_t downstream_length;
+ int err;
+
+ data->state = transform_state_finished;
+
+ for (;;) {
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+ data->zstrm.next_out = (unsigned char *) downstream_buffer;
+ data->zstrm.avail_out = downstream_length;
+
+ err = deflate(&data->zstrm, Z_FINISH);
+
+ if (downstream_length > (int64_t) data->zstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
+ data->downstream_length += (downstream_length - data->zstrm.avail_out);
+ }
+
+ if (err == Z_OK) { /* some more data to encode */
+ continue;
+ }
+
+ if (err != Z_STREAM_END) {
+ warning("deflate should report Z_STREAM_END");
+ }
+ break;
+ }
+
+ if (data->downstream_length != (int64_t) (data->zstrm.total_out)) {
+ error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
+ data->zstrm.total_out);
+ }
+
+ gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
+ }
+}
+
+
+static void
+gzip_transform_do(TSCont contp)
+{
+ TSVIO upstream_vio;
+ GzipData *data;
+ int64_t upstream_todo;
+ int64_t upstream_avail;
+ int64_t downstream_bytes_written;
+
+ data = (GzipData*)TSContDataGet(contp);
+ if (data->state == transform_state_initialized) {
+ gzip_transform_init(contp, data);
+ }
+
+ upstream_vio = TSVConnWriteVIOGet(contp);
+ downstream_bytes_written = data->downstream_length;
+
+ if (!TSVIOBufferGet(upstream_vio)) {
+ gzip_transform_finish(data);
+
+ TSVIONBytesSet(data->downstream_vio, data->downstream_length);
+
+ if (data->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(data->downstream_vio);
+ }
+ return;
+ }
+
+ upstream_todo = TSVIONTodoGet(upstream_vio);
+
+ if (upstream_todo > 0) {
+ upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
+
+ if (upstream_todo > upstream_avail) {
+ upstream_todo = upstream_avail;
+ }
+
+ if (upstream_todo > 0) {
+ gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
+ TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
+ }
+ }
+
+ if (TSVIONTodoGet(upstream_vio) > 0) {
+ if (upstream_todo > 0) {
+ if (data->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(data->downstream_vio);
+ }
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
+ }
+ } else {
+ gzip_transform_finish(data);
+ TSVIONBytesSet(data->downstream_vio, data->downstream_length);
+
+ if (data->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(data->downstream_vio);
+ }
+
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
+ }
+}
+
+
+static int
+gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
+{
+ if (TSVConnClosedGet(contp)) {
+ gzip_data_destroy((GzipData*)TSContDataGet(contp));
+ TSContDestroy(contp);
+ return 0;
+ } else {
+ switch (event) {
+ case TS_EVENT_ERROR:{
+ debug("gzip_transform: TS_EVENT_ERROR starts");
+ TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
+ }
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
+ break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ gzip_transform_do(contp);
+ break;
+ case TS_EVENT_IMMEDIATE:
+ gzip_transform_do(contp);
+ break;
+ default:
+ warning("unknown event [%d]", event);
+ gzip_transform_do(contp);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+
+static int
+gzip_transformable(TSHttpTxn txnp, int server, HostConfiguration * host_configuration, int *compress_type)
+{
+ /* Server response header */
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+ TSMLoc field_loc;
+
+ /* Client request header */
+ TSMBuffer cbuf;
+ TSMLoc chdr;
+ TSMLoc cfield;
+
+ const char *value;
+ int nvalues;
+ int i, compression_acceptable, len;
+
+ TSHttpStatus resp_status;
+ if (server) {
+ TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
+ } else {
+ TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
+ }
+ resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+
+ //conservatively pick some statusses to compress
+ if (!(resp_status == 200 || resp_status == 404 || resp_status == 500)) {
+ info("http response status [%d] is not compressible", resp_status);
+ return 0;
+ }
+
+ TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
+
+ //the only compressible method is currently GET.
+ int method_length;
+ const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length);
+ if (!(method_length == TS_HTTP_LEN_GET && memcmp(method, TS_HTTP_METHOD_GET, TS_HTTP_LEN_GET) == 0)) {
+ debug("method is not GET, not compressible");
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+ return 0;
+ }
+
+ cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ if (cfield != TS_NULL_MLOC) {
+ compression_acceptable = 0;
+ nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
+ for (i=0; i<nvalues; i++) {
+ value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len);
+ if (!value) {
+ continue;
+ }
+
+ if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
+ compression_acceptable = 1;
+ *compress_type = COMPRESSION_TYPE_DEFLATE;
+ break;
+ } else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) {
+ compression_acceptable = 1;
+ *compress_type = COMPRESSION_TYPE_GZIP;
+ break;
+ }
+ }
+
+ TSHandleMLocRelease(cbuf, chdr, cfield);
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+
+ if (!compression_acceptable) {
+ info("no acceptable encoding found in request header, not compressible");
+ return 0;
+ }
+ } else {
+ info("no acceptable encoding found in request header, not compressible");
+ TSHandleMLocRelease(cbuf, chdr, cfield);
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+ return 0;
+ }
+
+ if (server) {
+ TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
+ } else {
+ TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
+ }
+
+ /* If there already exists a content encoding then we don't want
+ to do anything. */
+ field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
+ if (field_loc) {
+ info("response is already content encoded, not compressible");
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return 0;
+ }
+
+ /* We only want to do gzip compression on documents that have a
+ content type of "text/" or "application/x-javascript". */
+ field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
+ if (!field_loc) {
+ info("no content type header found, not compressible");
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return 0;
+ }
+
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len);
+
+ int rv = host_configuration->ContentTypeIsCompressible(value, len);
+ if (!rv) {
+ info("content-type [%.*s] not compressible", len, value);
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return rv;
+}
+
+
+static void
+gzip_transform_add(TSHttpTxn txnp, int /* server ATS_UNUSED */, HostConfiguration * hc, int compress_type)
+{
+ int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked);
+ if (tmp) {
+ //happens on cache_stale_hit
+ debug("transform hook already set, bail");
+ return;
+ } else {
+ TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &GZIP_ONE);
+ info("adding compression transform");
+ }
+
+ TSHttpTxnUntransformedRespCache(txnp, 1);
+
+ if (!hc->cache()) {
+ debug("@@@@@ Gzip cache disabled");
+ TSHttpTxnTransformedRespCache(txnp, 0);
+ } else {
+ TSMBuffer bufp;
+ TSMLoc hdr_loc, field_loc;
+ int cache = 1;
+ if (TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS || TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) {
+ field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "@gzip_nocache", strlen("@gzip_nocache"));
+ if (field_loc) {
+ cache = 0;
+ debug("@@@@@ Gzip disallows cacheing of transformed response");
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ } else {
+ debug("@@@@ Gzip allows cacheing of transformed response");
+ }
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ }
+
+ TSHttpTxnTransformedRespCache(txnp, cache);
+ }
+
+ TSVConn connp;
+ GzipData *data;
+
+ connp = TSTransformCreate(gzip_transform, txnp);
+ data = gzip_data_alloc(compress_type);
+ data->txn = txnp;
+
+ TSContDataSet(connp, data);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
+}
+
+static int
+cache_transformable(TSHttpTxn txnp)
+{
+ int obj_status;
+
+ if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
+ warning("Couldn't get cache status of object");
+ return 0;
+ }
+ if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) {
+ info("stale cache hit");
+ return 0;
+ }
+ if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
+ info("fresh cache hit");
+ return 1;
+ }
+
+ return 0;
+}
+
+HostConfiguration *
+find_host_configuration(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer bufp, TSMLoc locp)
+{
+ TSMLoc fieldp = TSMimeHdrFieldFind(bufp, locp, TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST);
+
+ if (fieldp) {
+ int strl;
+ const char *strv = TSMimeHdrFieldValueStringGet(bufp, locp, fieldp, -1, &strl);
+ TSHandleMLocRelease(bufp, locp, fieldp);
+
+ HostConfiguration * host_configuration = config->Find(strv, strl);
+ return host_configuration;
+ }
+
+ return config->GlobalConfiguration();
+}
+
+
+static int
+transform_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata)
+{
+ TSHttpTxn txnp = (TSHttpTxn) edata;
+ int compress_type = COMPRESSION_TYPE_DEFLATE;
+
+ switch (event) {
+ case TS_EVENT_HTTP_READ_REQUEST_HDR:
+ {
+ TSMBuffer req_buf;
+ TSMLoc req_loc;
+ if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
+ int url_len;
+ char * url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
+ HostConfiguration * hc = find_host_configuration(txnp, req_buf, req_loc);
+ //we could clone the hosting configuration here, to make it deletable on reload?
+ TSHttpTxnArgSet(txnp, arg_idx_host_configuration, (void *) hc);
+
+ if (!hc->enabled() || !hc->IsUrlAllowed(url, url_len)) {
+ //FIXME: no double negatives
+ TSHttpTxnArgSet(txnp, arg_idx_url_disallowed, (void *) &GZIP_ONE);
+ info("url [%.*s] not allowed", url_len, url);
+ } else {
+ normalize_accept_encoding(txnp, req_buf, req_loc);
+ }
+ TSfree(url);
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ }
+ break;
+
+ case TS_EVENT_HTTP_READ_RESPONSE_HDR:
+ {
+ //os: the accept encoding header needs to be restored..
+ //otherwise the next request won't get a cache hit on this
+ HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
+ if (hc != NULL) {
+ if (hc->remove_accept_encoding()) {
+ TSMBuffer req_buf;
+ TSMLoc req_loc;
+ if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
+ restore_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ }
+ }
+
+ int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
+ if ( allowed && gzip_transformable(txnp, 1, hc, &compress_type)) {
+ gzip_transform_add(txnp, 1, hc, compress_type);
+ }
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ }
+ break;
+
+ case TS_EVENT_HTTP_SEND_REQUEST_HDR:
+ {
+ HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
+ if (hc!=NULL) {
+ if (hc->remove_accept_encoding()) {
+ TSMBuffer req_buf;
+ TSMLoc req_loc;
+ if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
+ hide_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ }
+ }
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ }
+ break;
+
+ case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
+ {
+ int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
+ HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
+ if ( hc != NULL ) {
+ if (allowed && cache_transformable(txnp) && gzip_transformable(txnp, 0, hc, &compress_type)) {
+ gzip_transform_add(txnp, 0, hc, compress_type);
+ }
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ }
+ break;
+
+ default:
+ fatal("gzip transform unknown event");
+ }
+
+ return 0;
+}
+
+
+static void
+read_configuration(TSCont contp) {
+ const char * path = (const char *)TSContDataGet(contp);
+ Configuration * newconfig = Configuration::Parse(path);
+
+ Configuration * oldconfig =__sync_lock_test_and_set(&config, newconfig);
+ debug("config swapped,old config %p", oldconfig);
+
+ //FIXME: we have leaked.
+ //consider cloning or refcounting the configuration passed to the txn
+ //to make deleting the old configuration possible
+ //if (config != NULL )
+ // delete config;
+}
+
+static int
+management_update(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
+{
+ TSReleaseAssert(event == TS_EVENT_MGMT_UPDATE);
+ info("management update event received");
+ read_configuration(contp);
+ return 0;
+}
+
+
+void
+TSPluginInit(int argc, const char *argv[])
+{
+ string config_path;
+
+ if (argc > 2) {
+ fatal("the gzip plugin does not accept more than 1 plugin argument");
+ } else if (argc == 2) {
+ config_path = std::string(argv[1]);
+ }
+
+ info("TSPluginInit %s", argv[0]);
+
+ if (!register_plugin()) {
+ fatal("The gzip plugin failed to register");
+ }
+
+ //if (argc == 2) {
+ // dictionary = load_dictionary(argv[1]);
+ //}
+
+ if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", &arg_idx_hooked) != TS_SUCCESS) {
+ fatal("failed to reserve an argument index");
+ }
+ if (TSHttpArgIndexReserve("gzip", "for storing if compression is applicable", &arg_idx_host_configuration) != TS_SUCCESS) {
+ fatal("failed to reserve an argument index");
+ }
+ if (TSHttpArgIndexReserve("gzip", "for storing if compression is disallowed for this txn", &arg_idx_url_disallowed) != TS_SUCCESS) {
+ fatal("failed to reserve an argument index");
+ }
+
+ global_hidden_header_name = init_hidden_header_name();
+
+ TSCont management_contp = TSContCreate(management_update, NULL);
+ //fixme: never freed. there is no shutdown event?
+ char * p = (char*)TSmalloc(config_path.size()+1);
+ strcpy(p,config_path.c_str());
+ TSContDataSet(management_contp,(void*)p);
+ TSMgmtUpdateRegister(management_contp, TAG);
+ read_configuration(management_contp);
+
+ TSCont transform_contp = TSContCreate(transform_plugin, NULL);
+ TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp);
+
+ info("loaded");
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.config
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/gzip.config b/plugins/experimental/ats_speed/gzip/gzip.config
new file mode 100644
index 0000000..81e9fa4
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/gzip.config
@@ -0,0 +1,6 @@
+# Set some global options first
+cache true
+enabled true
+remove-accept-encoding false
+compressible-content-type text/*
+compressible-content-type *javascript*
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/misc.cc b/plugins/experimental/ats_speed/gzip/misc.cc
new file mode 100644
index 0000000..0ea6911
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/misc.cc
@@ -0,0 +1,197 @@
+/** @file
+
+ Transforms content using gzip or deflate
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "ts/ts.h"
+//#include "ink_defs.h"
+#include <stdint.h>
+#include <inttypes.h>
+#include "misc.h"
+#include <string.h>
+#include "debug_macros.h"
+
+voidpf
+gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size)
+{
+ return (voidpf) TSmalloc(items * size);
+}
+
+void
+gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address)
+{
+ TSfree(address);
+}
+
+void
+normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ int deflate = 0;
+ int gzip = 0;
+
+ //remove the accept encoding field(s),
+ //while finding out if gzip or deflate is supported.
+ while (field) {
+ TSMLoc tmp;
+
+ if (!deflate && !gzip) {
+ int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field);
+
+ while (value_count > 0) {
+ int val_len = 0;
+ const char *val;
+
+ --value_count;
+ val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
+
+ if (val_len == (int) strlen("gzip"))
+ gzip = !strncmp(val, "gzip", val_len);
+ else if (val_len == (int) strlen("deflate"))
+ deflate = !strncmp(val, "deflate", val_len);
+ }
+ }
+
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+
+ //append a new accept-encoding field in the header
+ if (deflate || gzip) {
+ TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+
+ if (gzip) {
+ TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
+ info("normalized accept encoding to gzip");
+ } else if (deflate) {
+ TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", strlen("deflate"));
+ info("normalized accept encoding to deflate");
+ }
+
+ TSMimeHdrFieldAppend(reqp, hdr_loc, field);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ }
+}
+
+void
+hide_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ while (field) {
+ TSMLoc tmp;
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+}
+
+void
+restore_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
+
+ while (field) {
+ TSMLoc tmp;
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+}
+
+const char *
+init_hidden_header_name()
+{
+ char * hidden_header_name;
+ const char *var_name = "proxy.config.proxy_name";
+ TSMgmtString result;
+
+ if (TSMgmtStringGet(var_name, &result) != TS_SUCCESS) {
+ fatal("failed to get server name");
+ } else {
+ int hidden_header_name_len = strlen("x-accept-encoding-") + strlen(result);
+ hidden_header_name = (char *) TSmalloc(hidden_header_name_len + 1);
+ hidden_header_name[hidden_header_name_len] = 0;
+ sprintf(hidden_header_name, "x-accept-encoding-%s", result);
+ }
+ return hidden_header_name;
+}
+
+int
+register_plugin()
+{
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = (char*)"gzip";
+ info.vendor_name = (char*)"Apache";
+ info.support_email = (char*)"dev@trafficserver.apache.org";
+
+ if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
+ return 0;
+ }
+ return 1;
+}
+
+const char *
+load_dictionary(const char *preload_file)
+{
+ char *dict = (char *) malloc(800000);
+ uLong dictId = adler32(0L, Z_NULL, 0);
+ uLong *adler = &dictId;
+
+ FILE *fp;
+ int i = 0;
+
+ fp = fopen(preload_file, "r");
+ if (!fp) {
+ fatal("gzip-transform: ERROR: Unable to open dict file %s", preload_file);
+ }
+
+ /* dict = (char *) calloc(8000, sizeof(char)); */
+
+ i = 0;
+ while (!feof(fp)) {
+ if (fscanf(fp, "%s\n", dict + i) == 1) {
+ i = strlen(dict);
+ strcat(dict + i, " ");
+ ++i;
+ }
+ }
+ dict[i - 1] = '\0';
+
+ /* TODO get the adler compute right */
+ *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
+ return dict;
+}
+
+void
+gzip_log_ratio(int64_t in, int64_t out)
+{
+ // if (in) {
+ // info("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, ((float) (in - out) / in));
+ //} else {
+ // debug("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, 0.0F);
+ // }
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/misc.h b/plugins/experimental/ats_speed/gzip/misc.h
new file mode 100644
index 0000000..c44fb6b
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/misc.h
@@ -0,0 +1,84 @@
+/** @file
+
+ Transforms content using gzip or deflate
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#ifndef _GZIP_MISC_H_
+#define _GZIP_MISC_H_
+
+#define __STDC_LIMIT_MACROS
+#define __STDC_CONSTANT_MACROS
+#include <stdint.h>
+#include <zlib.h>
+#include <ts/ts.h>
+#include <stdlib.h> //exit()
+#include <stdio.h>
+
+//zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html
+static const int ZLIB_MEMLEVEL = 9; //min=1 (optimize for memory),max=9 (optimized for speed)
+static const int WINDOW_BITS_DEFLATE = -15;
+static const int WINDOW_BITS_GZIP = 31;
+
+//misc
+static const int COMPRESSION_TYPE_DEFLATE = 1;
+static const int COMPRESSION_TYPE_GZIP = 2;
+//this one is just for txnargset/get to point to
+static const int GZIP_ONE = 1;
+static const int DICT_PATH_MAX = 512;
+static const int DICT_ENTRY_MAX = 2048;
+
+//this one is used to rename the accept encoding header
+//it will be restored later on
+//to make it work, the name must be different then downstream proxies though
+//otherwise the downstream will restore the accept encoding header
+
+enum transform_state
+{
+ transform_state_initialized,
+ transform_state_output,
+ transform_state_finished
+};
+
+typedef struct
+{
+ TSHttpTxn txn;
+ TSVIO downstream_vio;
+ TSIOBuffer downstream_buffer;
+ TSIOBufferReader downstream_reader;
+ int downstream_length;
+ z_stream zstrm;
+ enum transform_state state;
+ int compression_type;
+} GzipData;
+
+
+voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
+void gzip_free(voidpf opaque, voidpf address);
+void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
+void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
+void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
+const char * init_hidden_header_name();
+int check_ts_version();
+int register_plugin();
+const char *load_dictionary(const char *preload_file);
+void gzip_log_ratio(int64_t in, int64_t out);
+
+#endif
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/scripts/prepare_psol.sh
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/scripts/prepare_psol.sh b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
new file mode 100755
index 0000000..4862249
--- /dev/null
+++ b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Copyright 2012 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Author: jefftk@google.com (Jeff Kaufman)
+# Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#
+# Usage:
+# scripts/prepare_psol.sh /path/to/mod_pagespeed/src
+#
+# Creates a directory psol/ and copies headers and a few source files from a
+# depot_tools (glient) checkout into psol/include. Along with creating
+# binaries, this is a step in preparing psol.tar.gz for distribution.
+#
+
+set -u # check for undefined variables
+set -e # exit on failed commands
+
+if [ "$(basename "$PWD")" != "ats_speed" ] ; then
+ echo "$(basename $0) must be invoked from the ats_speed directory"
+ exit 1
+fi
+
+if [ $# -ne 1 ] ; then
+ echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
+ exit 1
+fi
+
+MOD_PAGESPEED_SRC="$1"
+
+if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
+ basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
+ echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
+ exit 1
+fi
+
+if [ -e psol ] ; then
+ echo "A psol/ directory already exists. Move it somewhere else and rerun."
+ exit 1
+fi
+mkdir psol/
+# Copy over the .h files, plus a few selected .cc and .c files.
+rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
+ --exclude=".svn" \
+ --exclude=".git" \
+ --include='*.h' \
+ --include='*/' \
+ --include="apr_thread_compatible_pool.cc" \
+ --include="serf_url_async_fetcher.cc" \
+ --include="apr_mem_cache.cc" \
+ --include="key_value_codec.cc" \
+ --include="apr_memcache2.c" \
+ --include="loopback_route_fetcher.cc" \
+ --include="add_headers_fetcher.cc" \
+ --include="console_css_out.cc" \
+ --include="console_out.cc" \
+ --include="dense_hash_map" \
+ --include="dense_hash_set" \
+ --include="sparse_hash_map" \
+ --include="sparse_hash_set" \
+ --include="sparsetable" \
+ --include="mod_pagespeed_console_out.cc" \
+ --include="mod_pagespeed_console_css_out.cc" \
+ --include="mod_pagespeed_console_html_out.cc" \
+ --exclude='*'
+mkdir -p psol/lib/Debug/linux/ia32
+mkdir -p psol/lib/Debug/linux/x64
+mkdir -p psol/lib/Release/linux/ia32
+mkdir -p psol/lib/Release/linux/x64
+
+# Log that we did this.
+SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
+SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL | awk -F/ '{print $(NF-1)}')"
+
+DATE="$(date +%F)"
+echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
+ >> psol/include_history.txt
+
+echo
+echo "Output is in psol/include. Now put binaries in psol/lib following"
+echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
+echo "and then you can distribute PSOL."
+
Re: [1/3] ats_speed: PageSpeed optimization plugin
Posted by James Peach <jp...@apache.org>.
Great! Thanks a lot Otto :)
On Aug 4, 2014, at 6:57 AM, oschaaf@apache.org wrote:
> Repository: trafficserver
> Updated Branches:
> refs/heads/master fab202517 -> 083abd4ff
>
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/README
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/README b/plugins/experimental/ats_speed/gzip/README
> new file mode 100644
> index 0000000..2e74681
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/README
> @@ -0,0 +1,4 @@
> +This gzip is not compiled, but only here to be able to diff later
> +with the official gzip plugin. It is very slightly modified no
> +be able to disable caching of its compressed output through a
> +response header
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/configuration.cc b/plugins/experimental/ats_speed/gzip/configuration.cc
> new file mode 100644
> index 0000000..b1c499d
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/configuration.cc
> @@ -0,0 +1,264 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#include "configuration.h"
> +#include <fstream>
> +#include <algorithm>
> +#include <vector>
> +#include <fnmatch.h>
> +
> +namespace Gzip {
> + using namespace std;
> +
> + void ltrim_if(string& s, int (* fp) (int)) {
> + for (size_t i = 0; i < s.size();) {
> + if (fp(s[i])) {
> + s.erase(i,1);
> + } else {
> + break;
> + }
> + }
> + }
> +
> + void rtrim_if(string& s, int (* fp) (int)) {
> + for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) {
> + if (fp(s[i])) {
> + s.erase(i,1);
> + } else {
> + break;
> + }
> + }
> + }
> +
> + void trim_if(string& s, int (* fp) (int)) {
> + ltrim_if(s, fp);
> + rtrim_if(s, fp);
> + }
> +
> + vector<string> tokenize(const string &s, int (* fp) (int)) {
> + vector<string> r;
> + string tmp;
> +
> + for (size_t i = 0; i < s.size(); i++) {
> + if ( fp(s[i]) ) {
> + if ( tmp.size() ) {
> + r.push_back(tmp);
> + tmp = "";
> + }
> + } else {
> + tmp += s[i];
> + }
> + }
> +
> + if ( tmp.size() ) {
> + r.push_back(tmp);
> + }
> +
> + return r;
> + }
> +
> + enum ParserState {
> + kParseStart,
> + kParseCompressibleContentType,
> + kParseRemoveAcceptEncoding,
> + kParseEnable,
> + kParseCache,
> + kParseDisallow,
> + };
> +
> + void Configuration::AddHostConfiguration(HostConfiguration * hc){
> + host_configurations_.push_back(hc);
> + }
> +
> + void HostConfiguration::add_disallow(const std::string & disallow) {
> + disallows_.push_back(disallow);
> + }
> +
> + void HostConfiguration::add_compressible_content_type(const std::string & content_type) {
> + compressible_content_types_.push_back(content_type);
> + }
> +
> + HostConfiguration * Configuration::Find(const char * host, int host_length) {
> + HostConfiguration * host_configuration = host_configurations_[0];
> +
> + std::string shost(host, host_length);
> +
> + for (size_t i = 1; i < host_configurations_.size(); i++ ) {
> + if (host_configurations_[i]->host() == shost){
> + host_configuration = host_configurations_[i];
> + break;
> + }
> + }
> +
> + return host_configuration;
> + }
> +
> + bool HostConfiguration::IsUrlAllowed(const char * url, int url_len) {
> + string surl(url, url_len);
> +
> + for (size_t i = 0; i < disallows_.size(); i++) {
> + if ( fnmatch (disallows_[i].c_str(), surl.c_str(), 0) == 0 ) {
> + info("url [%s] disabled for compression, matched on pattern [%s]",
> + surl.c_str(), disallows_[i].c_str());
> + return false;
> + }
> + }
> +
> + return true;
> + }
> +
> + bool HostConfiguration::ContentTypeIsCompressible(const char * content_type, int content_type_length) {
> + string scontent_type(content_type, content_type_length);
> + bool is_match = false;
> +
> + for (size_t i = 0; i < compressible_content_types_.size(); i++) {
> + const char* match_string = compressible_content_types_[i].c_str();
> + bool exclude = match_string[0] == '!';
> + if (exclude) {
> + match_string++;//skip '!'
> + }
> + if ( fnmatch (match_string, scontent_type.c_str(), 0) == 0 ) {
> + info("compressible content type [%s], matched on pattern [%s]",
> + scontent_type.c_str(), compressible_content_types_[i].c_str());
> + is_match = !exclude;
> + }
> + }
> +
> + return is_match;
> + }
> +
> + Configuration * Configuration::Parse(const char * path ) {
> + string pathstring(path);
> +
> + // If we have a path and it's not an absolute path, make it relative to the
> + // configuration directory.
> + if (!pathstring.empty() && pathstring[0] != '/') {
> + pathstring.assign(TSConfigDirGet());
> + pathstring.append("/");
> + pathstring.append(path);
> + }
> +
> + trim_if(pathstring, isspace);
> +
> + Configuration * c = new Configuration();
> + HostConfiguration * current_host_configuration = new HostConfiguration("");
> + c->AddHostConfiguration(current_host_configuration);
> + current_host_configuration->add_compressible_content_type("text/*");
> + current_host_configuration->add_compressible_content_type("application/xml*");
> + current_host_configuration->add_compressible_content_type("*javascript*");
> + current_host_configuration->add_compressible_content_type("image/svg+xml");
> +
> +
> + if (pathstring.empty()) {
> + return c;
> + }
> +
> + path = pathstring.c_str();
> + info("Parsing file \"%s\"", path);
> + std::ifstream f;
> +
> + size_t lineno = 0;
> +
> + f.open(path, std::ios::in);
> +
> + if (!f.is_open()) {
> + warning("could not open file [%s], skip",path);
> + return c;
> + }
> +
> + enum ParserState state = kParseStart;
> +
> + while (!f.eof()) {
> + std::string line;
> + getline(f, line);
> + ++lineno;
> +
> + trim_if(line, isspace);
> + if (line.size() == 0) {
> + continue;
> + }
> +
> + vector<string> v = tokenize( line, isspace );
> +
> + for(size_t i = 0; i < v.size(); i++ ) {
> + string token = v[i];
> + trim_if(token, isspace);
> +
> + //should not happen
> + if (!token.size()) continue;
> +
> + //once a comment is encountered, we are done processing the line
> + if (token[0] == '#') break;
> +
> + switch(state) {
> + case kParseStart:
> + if ( (token[0] == '[') && (token[token.size()-1] == ']')){
> + std::string current_host = token.substr(1,token.size()-2);
> + current_host_configuration = new HostConfiguration(current_host);
> + c->AddHostConfiguration(current_host_configuration);
> + } else if (token == "compressible-content-type" ) {
> + state = kParseCompressibleContentType;
> + } else if (token == "remove-accept-encoding" ) {
> + state = kParseRemoveAcceptEncoding;
> + } else if (token == "enabled" ) {
> + state = kParseEnable;
> + } else if (token == "cache" ) {
> + state = kParseCache;
> + } else if (token == "disallow" ) {
> + state = kParseDisallow;
> + }
> + else {
> + warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno);
> + }
> + break;
> + case kParseCompressibleContentType:
> + current_host_configuration->add_compressible_content_type(token);
> + state = kParseStart;
> + break;
> + case kParseRemoveAcceptEncoding:
> + current_host_configuration->set_remove_accept_encoding(token == "true");
> + state = kParseStart;
> + break;
> + case kParseEnable:
> + current_host_configuration->set_enabled(token == "true");
> + state = kParseStart;
> + break;
> + case kParseCache:
> + current_host_configuration->set_cache(token == "true");
> + state = kParseStart;
> + break;
> + case kParseDisallow:
> + current_host_configuration->add_disallow(token);
> + state = kParseStart;
> + break;
> + }
> + }
> + }
> +
> + if (state != kParseStart) {
> + warning("the parser state indicates that data was expected when it reached the end of the file (%d)", state);
> + }
> +
> + return c;
> + } //Configuration::Parse
> +} //namespace
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/configuration.h b/plugins/experimental/ats_speed/gzip/configuration.h
> new file mode 100644
> index 0000000..b38cb64
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/configuration.h
> @@ -0,0 +1,84 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +
> +#ifndef GZIP_CONFIGURATION_H_
> +#define GZIP_CONFIGURATION_H_
> +
> +#include <string>
> +#include <vector>
> +#include "debug_macros.h"
> +
> +namespace Gzip {
> + class HostConfiguration {
> + public: //todo -> only configuration should be able to construct hostconfig
> + explicit HostConfiguration(const std::string & host)
> + : host_(host)
> + , enabled_(true)
> + , cache_(true)
> + , remove_accept_encoding_(false)
> + {}
> +
> + inline bool enabled() { return enabled_; }
> + inline void set_enabled(bool x) { enabled_ = x; }
> + inline bool cache() { return cache_; }
> + inline void set_cache(bool x) { cache_ = x; }
> + inline bool remove_accept_encoding() { return remove_accept_encoding_; }
> + inline void set_remove_accept_encoding(bool x) { remove_accept_encoding_ = x; }
> + inline std::string host() { return host_; }
> + void add_disallow(const std::string & disallow);
> + void add_compressible_content_type(const std::string & content_type);
> + bool IsUrlAllowed(const char * url, int url_len);
> + bool ContentTypeIsCompressible(const char * content_type, int content_type_length);
> +
> + private:
> + std::string host_;
> + bool enabled_;
> + bool cache_;
> + bool remove_accept_encoding_;
> + std::vector<std::string> compressible_content_types_;
> + std::vector<std::string> disallows_;
> + DISALLOW_COPY_AND_ASSIGN(HostConfiguration);
> + };//class HostConfiguration
> +
> + class Configuration {
> + friend class HostConfiguration;
> + public:
> + static Configuration * Parse(const char * path);
> + HostConfiguration * Find(const char * host, int host_length);
> + inline HostConfiguration * GlobalConfiguration() {
> + return host_configurations_[0];
> + }
> +
> + private:
> + explicit Configuration() {}
> + void AddHostConfiguration(HostConfiguration * hc);
> +
> + std::vector<HostConfiguration *> host_configurations_;
> + //todo: destructor. delete owned host configurations
> + DISALLOW_COPY_AND_ASSIGN(Configuration);
> + }; //class Configuration
> +
> +}//namespace
> +
> +#endif
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/debug_macros.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/debug_macros.h b/plugins/experimental/ats_speed/gzip/debug_macros.h
> new file mode 100644
> index 0000000..151de31
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/debug_macros.h
> @@ -0,0 +1,59 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#ifndef _DBG_MACROS_H
> +#define _DBG_MACROS_H
> +
> +#include <ts/ts.h>
> +
> +#define TAG "gzip"
> +
> +#define debug(fmt, args...) do { \
> + TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + } while (0)
> +
> +#define info(fmt, args...) do { \
> + TSDebug(TAG, "INFO: " fmt, ##args ); \
> + } while (0)
> +
> +#define warning(fmt, args...) do { \
> + TSDebug(TAG, "WARNING: " fmt, ##args ); \
> +} while (0)
> +
> +#define error(fmt, args...) do { \
> + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> +} while (0)
> +
> +#define fatal(fmt, args...) do { \
> + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + exit(-1); \
> +} while (0)
> +
> +//FIXME: this one doesn't deserve to be here
> +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
> + TypeName(const TypeName&); \
> + void operator=(const TypeName&)
> +
> +#endif //_DBG_MACROS_H
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/gzip.cc b/plugins/experimental/ats_speed/gzip/gzip.cc
> new file mode 100644
> index 0000000..1397762
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/gzip.cc
> @@ -0,0 +1,826 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +#define __STDC_LIMIT_MACROS
> +#define __STDC_CONSTANT_MACROS
> +#include <stdint.h>
> +
> +#include <string>
> +#include <string.h>
> +#include <zlib.h>
> +#include <ts/ts.h>
> +#include "debug_macros.h"
> +#include "misc.h"
> +#include "configuration.h"
> +//#include "ink_defs.h"
> +
> +using namespace std;
> +using namespace Gzip;
> +
> +//FIXME: custom dictionaries would be nice. configurable/content-type?
> +//FIXME: look into autoscaling the compression level based on connection speed
> +// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
> +// might be served better with little or no compression at all
> +//FIXME: look into compressing from the task thread pool
> +//FIXME: make normalizing accept encoding configurable
> +
> +// from mod_deflate:
> +// ZLIB's compression algorithm uses a
> +// 0-9 based scale that GZIP does where '1' is 'Best speed'
> +// and '9' is 'Best compression'. Testing has proved level '6'
> +// to be about the best level to use in an HTTP Server.
> +
> +const int ZLIB_COMPRESSION_LEVEL = 6;
> +
> +int arg_idx_hooked;
> +int arg_idx_host_configuration;
> +int arg_idx_url_disallowed;
> +
> +
> +const char * global_hidden_header_name;
> +Configuration* config = NULL;
> +const char *dictionary = NULL;
> +
> +static GzipData *
> +gzip_data_alloc(int compression_type)
> +{
> + GzipData *data;
> + int err;
> +
> + data = (GzipData *) TSmalloc(sizeof(GzipData));
> + data->downstream_vio = NULL;
> + data->downstream_buffer = NULL;
> + data->downstream_reader = NULL;
> + data->downstream_length = 0;
> + data->state = transform_state_initialized;
> + data->compression_type = compression_type;
> + data->zstrm.next_in = Z_NULL;
> + data->zstrm.avail_in = 0;
> + data->zstrm.total_in = 0;
> + data->zstrm.next_out = Z_NULL;
> + data->zstrm.avail_out = 0;
> + data->zstrm.total_out = 0;
> + data->zstrm.zalloc = gzip_alloc;
> + data->zstrm.zfree = gzip_free;
> + data->zstrm.opaque = (voidpf) 0;
> + data->zstrm.data_type = Z_ASCII;
> +
> + int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
> +
> + err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
> +
> + if (err != Z_OK) {
> + fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
> + }
> +
> + if (dictionary) {
> + err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
> + if (err != Z_OK) {
> + fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
> + }
> + }
> +
> + return data;
> +}
> +
> +
> +static void
> +gzip_data_destroy(GzipData * data)
> +{
> + TSReleaseAssert(data);
> +
> + //deflateEnd returnvalue ignore is intentional
> + //it would spew log on every client abort
> + deflateEnd(&data->zstrm);
> +
> + if (data->downstream_buffer) {
> + TSIOBufferDestroy(data->downstream_buffer);
> + }
> +
> + TSfree(data);
> +}
> +
> +static TSReturnCode
> +gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type)
> +{
> + TSReturnCode ret;
> + TSMLoc ce_loc;
> +
> + // Delete Content-Encoding if present???
> +
> + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Content-Encoding", sizeof("Content-Encoding") - 1, &ce_loc)) == TS_SUCCESS) {
> + if (compression_type == COMPRESSION_TYPE_DEFLATE) {
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", sizeof("deflate") - 1);
> + } else if (compression_type == COMPRESSION_TYPE_GZIP) {
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", sizeof("gzip") - 1);
> + }
> + if (ret == TS_SUCCESS) {
> + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot add the Content-Encoding header");
> + }
> +
> + return ret;
> +}
> +
> +static TSReturnCode
> +gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
> +{
> + TSReturnCode ret;
> + TSMLoc ce_loc;
> +
> + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Vary", sizeof("Vary") - 1);
> + if (ce_loc) {
> + int idx, count, len;
> + const char *value;
> +
> + count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc);
> + for(idx=0; idx<count; idx++) {
> + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len);
> + if (len &&
> + strncasecmp("Accept-Encoding", value, len) == 0) {
> + // Bail, Vary: Accept-Encoding already sent from origin
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + return TS_SUCCESS;
> + }
> + }
> +
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1);
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + } else {
> + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Vary", sizeof("Vary") - 1, &ce_loc)) == TS_SUCCESS) {
> + if ((ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1)) == TS_SUCCESS) {
> + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
> + }
> +
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot add/update the Vary header");
> + }
> +
> + return ret;
> +}
> +
> +//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
> +// specify a very header..
> +static TSReturnCode
> +gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
> +{
> + TSReturnCode ret = TS_SUCCESS;
> + TSMLoc ce_loc;
> +
> + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
> +
> + if (ce_loc) {
> + int changetag = 1;
> + int strl;
> + const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
> + //do not alter weak etags.
> + //FIXME: consider just making the etag weak for compressed content
> + if (strl >= 2) {
> + if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
> + changetag = 0;
> + }
> + if (changetag) {
> + ret = TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
> + }
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot handle the %s header", TS_MIME_FIELD_ETAG);
> + }
> +
> + return ret;
> +}
> +
> +//FIXME: some things are potentially compressible. those responses
> +static void
> +gzip_transform_init(TSCont contp, GzipData * data)
> +{
> + //update the vary, content-encoding, and etag response headers
> + //prepare the downstream for transforming
> +
> + TSVConn downstream_conn;
> + TSMBuffer bufp;
> + TSMLoc hdr_loc;
> +
> + data->state = transform_state_output;
> +
> + if (TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc) != TS_SUCCESS) {
> + error("Error TSHttpTxnTransformRespGet");
> + return;
> + }
> +
> + if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == TS_SUCCESS &&
> + gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS &&
> + gzip_etag_header(bufp, hdr_loc) == TS_SUCCESS) {
> + downstream_conn = TSTransformOutputVConnGet(contp);
> + data->downstream_buffer = TSIOBufferCreate();
> + data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
> + data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
> + }
> +
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> +}
> +
> +
> +
> +static void
> +gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
> +{
> + TSIOBufferBlock downstream_blkp;
> + const char *upstream_buffer;
> + char *downstream_buffer;
> + int64_t upstream_length, downstream_length;
> + int err;
> +
> + while (amount > 0) {
> + downstream_blkp = TSIOBufferReaderStart(upstream_reader);
> + if (!downstream_blkp) {
> + error("couldn't get from IOBufferBlock");
> + return;
> + }
> +
> + upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
> + if (!upstream_buffer) {
> + error("couldn't get from TSIOBufferBlockReadStart");
> + return;
> + }
> +
> + if (upstream_length > amount) {
> + upstream_length = amount;
> + }
> +
> + data->zstrm.next_in = (unsigned char *) upstream_buffer;
> + data->zstrm.avail_in = upstream_length;
> +
> + while (data->zstrm.avail_in > 0) {
> + downstream_blkp = TSIOBufferStart(data->downstream_buffer);
> + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
> +
> + data->zstrm.next_out = (unsigned char *) downstream_buffer;
> + data->zstrm.avail_out = downstream_length;
> +
> + err = deflate(&data->zstrm, Z_NO_FLUSH);
> +
> + if (err != Z_OK)
> + warning("deflate() call failed: %d", err);
> +
> + if (downstream_length > data->zstrm.avail_out) {
> + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
> + data->downstream_length += (downstream_length - data->zstrm.avail_out);
> + }
> +
> + if (data->zstrm.avail_out > 0) {
> + if (data->zstrm.avail_in != 0) {
> + error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
> + }
> + }
> + }
> +
> + TSIOBufferReaderConsume(upstream_reader, upstream_length);
> + amount -= upstream_length;
> + }
> +}
> +
> +static void
> +gzip_transform_finish(GzipData * data)
> +{
> + if (data->state == transform_state_output) {
> + TSIOBufferBlock downstream_blkp;
> + char *downstream_buffer;
> + int64_t downstream_length;
> + int err;
> +
> + data->state = transform_state_finished;
> +
> + for (;;) {
> + downstream_blkp = TSIOBufferStart(data->downstream_buffer);
> +
> + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
> + data->zstrm.next_out = (unsigned char *) downstream_buffer;
> + data->zstrm.avail_out = downstream_length;
> +
> + err = deflate(&data->zstrm, Z_FINISH);
> +
> + if (downstream_length > (int64_t) data->zstrm.avail_out) {
> + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
> + data->downstream_length += (downstream_length - data->zstrm.avail_out);
> + }
> +
> + if (err == Z_OK) { /* some more data to encode */
> + continue;
> + }
> +
> + if (err != Z_STREAM_END) {
> + warning("deflate should report Z_STREAM_END");
> + }
> + break;
> + }
> +
> + if (data->downstream_length != (int64_t) (data->zstrm.total_out)) {
> + error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
> + data->zstrm.total_out);
> + }
> +
> + gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
> + }
> +}
> +
> +
> +static void
> +gzip_transform_do(TSCont contp)
> +{
> + TSVIO upstream_vio;
> + GzipData *data;
> + int64_t upstream_todo;
> + int64_t upstream_avail;
> + int64_t downstream_bytes_written;
> +
> + data = (GzipData*)TSContDataGet(contp);
> + if (data->state == transform_state_initialized) {
> + gzip_transform_init(contp, data);
> + }
> +
> + upstream_vio = TSVConnWriteVIOGet(contp);
> + downstream_bytes_written = data->downstream_length;
> +
> + if (!TSVIOBufferGet(upstream_vio)) {
> + gzip_transform_finish(data);
> +
> + TSVIONBytesSet(data->downstream_vio, data->downstream_length);
> +
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> + return;
> + }
> +
> + upstream_todo = TSVIONTodoGet(upstream_vio);
> +
> + if (upstream_todo > 0) {
> + upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
> +
> + if (upstream_todo > upstream_avail) {
> + upstream_todo = upstream_avail;
> + }
> +
> + if (upstream_todo > 0) {
> + gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
> + TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
> + }
> + }
> +
> + if (TSVIONTodoGet(upstream_vio) > 0) {
> + if (upstream_todo > 0) {
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
> + }
> + } else {
> + gzip_transform_finish(data);
> + TSVIONBytesSet(data->downstream_vio, data->downstream_length);
> +
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> +
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
> + }
> +}
> +
> +
> +static int
> +gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
> +{
> + if (TSVConnClosedGet(contp)) {
> + gzip_data_destroy((GzipData*)TSContDataGet(contp));
> + TSContDestroy(contp);
> + return 0;
> + } else {
> + switch (event) {
> + case TS_EVENT_ERROR:{
> + debug("gzip_transform: TS_EVENT_ERROR starts");
> + TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
> + }
> + break;
> + case TS_EVENT_VCONN_WRITE_COMPLETE:
> + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
> + break;
> + case TS_EVENT_VCONN_WRITE_READY:
> + gzip_transform_do(contp);
> + break;
> + case TS_EVENT_IMMEDIATE:
> + gzip_transform_do(contp);
> + break;
> + default:
> + warning("unknown event [%d]", event);
> + gzip_transform_do(contp);
> + break;
> + }
> + }
> +
> + return 0;
> +}
> +
> +
> +static int
> +gzip_transformable(TSHttpTxn txnp, int server, HostConfiguration * host_configuration, int *compress_type)
> +{
> + /* Server response header */
> + TSMBuffer bufp;
> + TSMLoc hdr_loc;
> + TSMLoc field_loc;
> +
> + /* Client request header */
> + TSMBuffer cbuf;
> + TSMLoc chdr;
> + TSMLoc cfield;
> +
> + const char *value;
> + int nvalues;
> + int i, compression_acceptable, len;
> +
> + TSHttpStatus resp_status;
> + if (server) {
> + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
> + } else {
> + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
> + }
> + resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> +
> + //conservatively pick some statusses to compress
> + if (!(resp_status == 200 || resp_status == 404 || resp_status == 500)) {
> + info("http response status [%d] is not compressible", resp_status);
> + return 0;
> + }
> +
> + TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
> +
> + //the only compressible method is currently GET.
> + int method_length;
> + const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length);
> + if (!(method_length == TS_HTTP_LEN_GET && memcmp(method, TS_HTTP_METHOD_GET, TS_HTTP_LEN_GET) == 0)) {
> + debug("method is not GET, not compressible");
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> + return 0;
> + }
> +
> + cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + if (cfield != TS_NULL_MLOC) {
> + compression_acceptable = 0;
> + nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
> + for (i=0; i<nvalues; i++) {
> + value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len);
> + if (!value) {
> + continue;
> + }
> +
> + if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
> + compression_acceptable = 1;
> + *compress_type = COMPRESSION_TYPE_DEFLATE;
> + break;
> + } else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) {
> + compression_acceptable = 1;
> + *compress_type = COMPRESSION_TYPE_GZIP;
> + break;
> + }
> + }
> +
> + TSHandleMLocRelease(cbuf, chdr, cfield);
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> +
> + if (!compression_acceptable) {
> + info("no acceptable encoding found in request header, not compressible");
> + return 0;
> + }
> + } else {
> + info("no acceptable encoding found in request header, not compressible");
> + TSHandleMLocRelease(cbuf, chdr, cfield);
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> + return 0;
> + }
> +
> + if (server) {
> + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
> + } else {
> + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
> + }
> +
> + /* If there already exists a content encoding then we don't want
> + to do anything. */
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
> + if (field_loc) {
> + info("response is already content encoded, not compressible");
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return 0;
> + }
> +
> + /* We only want to do gzip compression on documents that have a
> + content type of "text/" or "application/x-javascript". */
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
> + if (!field_loc) {
> + info("no content type header found, not compressible");
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return 0;
> + }
> +
> + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len);
> +
> + int rv = host_configuration->ContentTypeIsCompressible(value, len);
> + if (!rv) {
> + info("content-type [%.*s] not compressible", len, value);
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return rv;
> +}
> +
> +
> +static void
> +gzip_transform_add(TSHttpTxn txnp, int /* server ATS_UNUSED */, HostConfiguration * hc, int compress_type)
> +{
> + int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked);
> + if (tmp) {
> + //happens on cache_stale_hit
> + debug("transform hook already set, bail");
> + return;
> + } else {
> + TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &GZIP_ONE);
> + info("adding compression transform");
> + }
> +
> + TSHttpTxnUntransformedRespCache(txnp, 1);
> +
> + if (!hc->cache()) {
> + debug("@@@@@ Gzip cache disabled");
> + TSHttpTxnTransformedRespCache(txnp, 0);
> + } else {
> + TSMBuffer bufp;
> + TSMLoc hdr_loc, field_loc;
> + int cache = 1;
> + if (TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS || TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) {
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "@gzip_nocache", strlen("@gzip_nocache"));
> + if (field_loc) {
> + cache = 0;
> + debug("@@@@@ Gzip disallows cacheing of transformed response");
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + } else {
> + debug("@@@@ Gzip allows cacheing of transformed response");
> + }
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + }
> +
> + TSHttpTxnTransformedRespCache(txnp, cache);
> + }
> +
> + TSVConn connp;
> + GzipData *data;
> +
> + connp = TSTransformCreate(gzip_transform, txnp);
> + data = gzip_data_alloc(compress_type);
> + data->txn = txnp;
> +
> + TSContDataSet(connp, data);
> + TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
> +}
> +
> +static int
> +cache_transformable(TSHttpTxn txnp)
> +{
> + int obj_status;
> +
> + if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
> + warning("Couldn't get cache status of object");
> + return 0;
> + }
> + if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) {
> + info("stale cache hit");
> + return 0;
> + }
> + if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
> + info("fresh cache hit");
> + return 1;
> + }
> +
> + return 0;
> +}
> +
> +HostConfiguration *
> +find_host_configuration(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer bufp, TSMLoc locp)
> +{
> + TSMLoc fieldp = TSMimeHdrFieldFind(bufp, locp, TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST);
> +
> + if (fieldp) {
> + int strl;
> + const char *strv = TSMimeHdrFieldValueStringGet(bufp, locp, fieldp, -1, &strl);
> + TSHandleMLocRelease(bufp, locp, fieldp);
> +
> + HostConfiguration * host_configuration = config->Find(strv, strl);
> + return host_configuration;
> + }
> +
> + return config->GlobalConfiguration();
> +}
> +
> +
> +static int
> +transform_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata)
> +{
> + TSHttpTxn txnp = (TSHttpTxn) edata;
> + int compress_type = COMPRESSION_TYPE_DEFLATE;
> +
> + switch (event) {
> + case TS_EVENT_HTTP_READ_REQUEST_HDR:
> + {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + int url_len;
> + char * url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
> + HostConfiguration * hc = find_host_configuration(txnp, req_buf, req_loc);
> + //we could clone the hosting configuration here, to make it deletable on reload?
> + TSHttpTxnArgSet(txnp, arg_idx_host_configuration, (void *) hc);
> +
> + if (!hc->enabled() || !hc->IsUrlAllowed(url, url_len)) {
> + //FIXME: no double negatives
> + TSHttpTxnArgSet(txnp, arg_idx_url_disallowed, (void *) &GZIP_ONE);
> + info("url [%.*s] not allowed", url_len, url);
> + } else {
> + normalize_accept_encoding(txnp, req_buf, req_loc);
> + }
> + TSfree(url);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_READ_RESPONSE_HDR:
> + {
> + //os: the accept encoding header needs to be restored..
> + //otherwise the next request won't get a cache hit on this
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if (hc != NULL) {
> + if (hc->remove_accept_encoding()) {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + restore_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + }
> +
> + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
> + if ( allowed && gzip_transformable(txnp, 1, hc, &compress_type)) {
> + gzip_transform_add(txnp, 1, hc, compress_type);
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_SEND_REQUEST_HDR:
> + {
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if (hc!=NULL) {
> + if (hc->remove_accept_encoding()) {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + hide_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
> + {
> + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if ( hc != NULL ) {
> + if (allowed && cache_transformable(txnp) && gzip_transformable(txnp, 0, hc, &compress_type)) {
> + gzip_transform_add(txnp, 0, hc, compress_type);
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + default:
> + fatal("gzip transform unknown event");
> + }
> +
> + return 0;
> +}
> +
> +
> +static void
> +read_configuration(TSCont contp) {
> + const char * path = (const char *)TSContDataGet(contp);
> + Configuration * newconfig = Configuration::Parse(path);
> +
> + Configuration * oldconfig =__sync_lock_test_and_set(&config, newconfig);
> + debug("config swapped,old config %p", oldconfig);
> +
> + //FIXME: we have leaked.
> + //consider cloning or refcounting the configuration passed to the txn
> + //to make deleting the old configuration possible
> + //if (config != NULL )
> + // delete config;
> +}
> +
> +static int
> +management_update(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
> +{
> + TSReleaseAssert(event == TS_EVENT_MGMT_UPDATE);
> + info("management update event received");
> + read_configuration(contp);
> + return 0;
> +}
> +
> +
> +void
> +TSPluginInit(int argc, const char *argv[])
> +{
> + string config_path;
> +
> + if (argc > 2) {
> + fatal("the gzip plugin does not accept more than 1 plugin argument");
> + } else if (argc == 2) {
> + config_path = std::string(argv[1]);
> + }
> +
> + info("TSPluginInit %s", argv[0]);
> +
> + if (!register_plugin()) {
> + fatal("The gzip plugin failed to register");
> + }
> +
> + //if (argc == 2) {
> + // dictionary = load_dictionary(argv[1]);
> + //}
> +
> + if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", &arg_idx_hooked) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> + if (TSHttpArgIndexReserve("gzip", "for storing if compression is applicable", &arg_idx_host_configuration) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> + if (TSHttpArgIndexReserve("gzip", "for storing if compression is disallowed for this txn", &arg_idx_url_disallowed) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> +
> + global_hidden_header_name = init_hidden_header_name();
> +
> + TSCont management_contp = TSContCreate(management_update, NULL);
> + //fixme: never freed. there is no shutdown event?
> + char * p = (char*)TSmalloc(config_path.size()+1);
> + strcpy(p,config_path.c_str());
> + TSContDataSet(management_contp,(void*)p);
> + TSMgmtUpdateRegister(management_contp, TAG);
> + read_configuration(management_contp);
> +
> + TSCont transform_contp = TSContCreate(transform_plugin, NULL);
> + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp);
> +
> + info("loaded");
> +}
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.config
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/gzip.config b/plugins/experimental/ats_speed/gzip/gzip.config
> new file mode 100644
> index 0000000..81e9fa4
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/gzip.config
> @@ -0,0 +1,6 @@
> +# Set some global options first
> +cache true
> +enabled true
> +remove-accept-encoding false
> +compressible-content-type text/*
> +compressible-content-type *javascript*
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/misc.cc b/plugins/experimental/ats_speed/gzip/misc.cc
> new file mode 100644
> index 0000000..0ea6911
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/misc.cc
> @@ -0,0 +1,197 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#include "ts/ts.h"
> +//#include "ink_defs.h"
> +#include <stdint.h>
> +#include <inttypes.h>
> +#include "misc.h"
> +#include <string.h>
> +#include "debug_macros.h"
> +
> +voidpf
> +gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size)
> +{
> + return (voidpf) TSmalloc(items * size);
> +}
> +
> +void
> +gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address)
> +{
> + TSfree(address);
> +}
> +
> +void
> +normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + int deflate = 0;
> + int gzip = 0;
> +
> + //remove the accept encoding field(s),
> + //while finding out if gzip or deflate is supported.
> + while (field) {
> + TSMLoc tmp;
> +
> + if (!deflate && !gzip) {
> + int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field);
> +
> + while (value_count > 0) {
> + int val_len = 0;
> + const char *val;
> +
> + --value_count;
> + val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
> +
> + if (val_len == (int) strlen("gzip"))
> + gzip = !strncmp(val, "gzip", val_len);
> + else if (val_len == (int) strlen("deflate"))
> + deflate = !strncmp(val, "deflate", val_len);
> + }
> + }
> +
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +
> + //append a new accept-encoding field in the header
> + if (deflate || gzip) {
> + TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> +
> + if (gzip) {
> + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
> + info("normalized accept encoding to gzip");
> + } else if (deflate) {
> + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", strlen("deflate"));
> + info("normalized accept encoding to deflate");
> + }
> +
> + TSMimeHdrFieldAppend(reqp, hdr_loc, field);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + }
> +}
> +
> +void
> +hide_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + while (field) {
> + TSMLoc tmp;
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +}
> +
> +void
> +restore_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
> +
> + while (field) {
> + TSMLoc tmp;
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +}
> +
> +const char *
> +init_hidden_header_name()
> +{
> + char * hidden_header_name;
> + const char *var_name = "proxy.config.proxy_name";
> + TSMgmtString result;
> +
> + if (TSMgmtStringGet(var_name, &result) != TS_SUCCESS) {
> + fatal("failed to get server name");
> + } else {
> + int hidden_header_name_len = strlen("x-accept-encoding-") + strlen(result);
> + hidden_header_name = (char *) TSmalloc(hidden_header_name_len + 1);
> + hidden_header_name[hidden_header_name_len] = 0;
> + sprintf(hidden_header_name, "x-accept-encoding-%s", result);
> + }
> + return hidden_header_name;
> +}
> +
> +int
> +register_plugin()
> +{
> + TSPluginRegistrationInfo info;
> +
> + info.plugin_name = (char*)"gzip";
> + info.vendor_name = (char*)"Apache";
> + info.support_email = (char*)"dev@trafficserver.apache.org";
> +
> + if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
> + return 0;
> + }
> + return 1;
> +}
> +
> +const char *
> +load_dictionary(const char *preload_file)
> +{
> + char *dict = (char *) malloc(800000);
> + uLong dictId = adler32(0L, Z_NULL, 0);
> + uLong *adler = &dictId;
> +
> + FILE *fp;
> + int i = 0;
> +
> + fp = fopen(preload_file, "r");
> + if (!fp) {
> + fatal("gzip-transform: ERROR: Unable to open dict file %s", preload_file);
> + }
> +
> + /* dict = (char *) calloc(8000, sizeof(char)); */
> +
> + i = 0;
> + while (!feof(fp)) {
> + if (fscanf(fp, "%s\n", dict + i) == 1) {
> + i = strlen(dict);
> + strcat(dict + i, " ");
> + ++i;
> + }
> + }
> + dict[i - 1] = '\0';
> +
> + /* TODO get the adler compute right */
> + *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
> + return dict;
> +}
> +
> +void
> +gzip_log_ratio(int64_t in, int64_t out)
> +{
> + // if (in) {
> + // info("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, ((float) (in - out) / in));
> + //} else {
> + // debug("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, 0.0F);
> + // }
> +}
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/misc.h b/plugins/experimental/ats_speed/gzip/misc.h
> new file mode 100644
> index 0000000..c44fb6b
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/misc.h
> @@ -0,0 +1,84 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#ifndef _GZIP_MISC_H_
> +#define _GZIP_MISC_H_
> +
> +#define __STDC_LIMIT_MACROS
> +#define __STDC_CONSTANT_MACROS
> +#include <stdint.h>
> +#include <zlib.h>
> +#include <ts/ts.h>
> +#include <stdlib.h> //exit()
> +#include <stdio.h>
> +
> +//zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html
> +static const int ZLIB_MEMLEVEL = 9; //min=1 (optimize for memory),max=9 (optimized for speed)
> +static const int WINDOW_BITS_DEFLATE = -15;
> +static const int WINDOW_BITS_GZIP = 31;
> +
> +//misc
> +static const int COMPRESSION_TYPE_DEFLATE = 1;
> +static const int COMPRESSION_TYPE_GZIP = 2;
> +//this one is just for txnargset/get to point to
> +static const int GZIP_ONE = 1;
> +static const int DICT_PATH_MAX = 512;
> +static const int DICT_ENTRY_MAX = 2048;
> +
> +//this one is used to rename the accept encoding header
> +//it will be restored later on
> +//to make it work, the name must be different then downstream proxies though
> +//otherwise the downstream will restore the accept encoding header
> +
> +enum transform_state
> +{
> + transform_state_initialized,
> + transform_state_output,
> + transform_state_finished
> +};
> +
> +typedef struct
> +{
> + TSHttpTxn txn;
> + TSVIO downstream_vio;
> + TSIOBuffer downstream_buffer;
> + TSIOBufferReader downstream_reader;
> + int downstream_length;
> + z_stream zstrm;
> + enum transform_state state;
> + int compression_type;
> +} GzipData;
> +
> +
> +voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
> +void gzip_free(voidpf opaque, voidpf address);
> +void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
> +void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
> +void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
> +const char * init_hidden_header_name();
> +int check_ts_version();
> +int register_plugin();
> +const char *load_dictionary(const char *preload_file);
> +void gzip_log_ratio(int64_t in, int64_t out);
> +
> +#endif
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/scripts/prepare_psol.sh b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> new file mode 100755
> index 0000000..4862249
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> @@ -0,0 +1,95 @@
> +#!/bin/bash
> +#
> +# Copyright 2012 Google Inc.
> +#
> +# Licensed under the Apache License, Version 2.0 (the "License");
> +# you may not use this file except in compliance with the License.
> +# You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +#
> +# Author: jefftk@google.com (Jeff Kaufman)
> +# Author: oschaaf@we-amp.com (Otto van der Schaaf)
> +#
> +# Usage:
> +# scripts/prepare_psol.sh /path/to/mod_pagespeed/src
> +#
> +# Creates a directory psol/ and copies headers and a few source files from a
> +# depot_tools (glient) checkout into psol/include. Along with creating
> +# binaries, this is a step in preparing psol.tar.gz for distribution.
> +#
> +
> +set -u # check for undefined variables
> +set -e # exit on failed commands
> +
> +if [ "$(basename "$PWD")" != "ats_speed" ] ; then
> + echo "$(basename $0) must be invoked from the ats_speed directory"
> + exit 1
> +fi
> +
> +if [ $# -ne 1 ] ; then
> + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
> + exit 1
> +fi
> +
> +MOD_PAGESPEED_SRC="$1"
> +
> +if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
> + basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
> + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
> + exit 1
> +fi
> +
> +if [ -e psol ] ; then
> + echo "A psol/ directory already exists. Move it somewhere else and rerun."
> + exit 1
> +fi
> +mkdir psol/
> +# Copy over the .h files, plus a few selected .cc and .c files.
> +rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
> + --exclude=".svn" \
> + --exclude=".git" \
> + --include='*.h' \
> + --include='*/' \
> + --include="apr_thread_compatible_pool.cc" \
> + --include="serf_url_async_fetcher.cc" \
> + --include="apr_mem_cache.cc" \
> + --include="key_value_codec.cc" \
> + --include="apr_memcache2.c" \
> + --include="loopback_route_fetcher.cc" \
> + --include="add_headers_fetcher.cc" \
> + --include="console_css_out.cc" \
> + --include="console_out.cc" \
> + --include="dense_hash_map" \
> + --include="dense_hash_set" \
> + --include="sparse_hash_map" \
> + --include="sparse_hash_set" \
> + --include="sparsetable" \
> + --include="mod_pagespeed_console_out.cc" \
> + --include="mod_pagespeed_console_css_out.cc" \
> + --include="mod_pagespeed_console_html_out.cc" \
> + --exclude='*'
> +mkdir -p psol/lib/Debug/linux/ia32
> +mkdir -p psol/lib/Debug/linux/x64
> +mkdir -p psol/lib/Release/linux/ia32
> +mkdir -p psol/lib/Release/linux/x64
> +
> +# Log that we did this.
> +SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
> +SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL | awk -F/ '{print $(NF-1)}')"
> +
> +DATE="$(date +%F)"
> +echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
> + >> psol/include_history.txt
> +
> +echo
> +echo "Output is in psol/include. Now put binaries in psol/lib following"
> +echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
> +echo "and then you can distribute PSOL."
> +
>
Re: [1/3] ats_speed: PageSpeed optimization plugin
Posted by James Peach <jp...@apache.org>.
Great! Thanks a lot Otto :)
On Aug 4, 2014, at 6:57 AM, oschaaf@apache.org wrote:
> Repository: trafficserver
> Updated Branches:
> refs/heads/master fab202517 -> 083abd4ff
>
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/README
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/README b/plugins/experimental/ats_speed/gzip/README
> new file mode 100644
> index 0000000..2e74681
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/README
> @@ -0,0 +1,4 @@
> +This gzip is not compiled, but only here to be able to diff later
> +with the official gzip plugin. It is very slightly modified no
> +be able to disable caching of its compressed output through a
> +response header
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/configuration.cc b/plugins/experimental/ats_speed/gzip/configuration.cc
> new file mode 100644
> index 0000000..b1c499d
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/configuration.cc
> @@ -0,0 +1,264 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#include "configuration.h"
> +#include <fstream>
> +#include <algorithm>
> +#include <vector>
> +#include <fnmatch.h>
> +
> +namespace Gzip {
> + using namespace std;
> +
> + void ltrim_if(string& s, int (* fp) (int)) {
> + for (size_t i = 0; i < s.size();) {
> + if (fp(s[i])) {
> + s.erase(i,1);
> + } else {
> + break;
> + }
> + }
> + }
> +
> + void rtrim_if(string& s, int (* fp) (int)) {
> + for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) {
> + if (fp(s[i])) {
> + s.erase(i,1);
> + } else {
> + break;
> + }
> + }
> + }
> +
> + void trim_if(string& s, int (* fp) (int)) {
> + ltrim_if(s, fp);
> + rtrim_if(s, fp);
> + }
> +
> + vector<string> tokenize(const string &s, int (* fp) (int)) {
> + vector<string> r;
> + string tmp;
> +
> + for (size_t i = 0; i < s.size(); i++) {
> + if ( fp(s[i]) ) {
> + if ( tmp.size() ) {
> + r.push_back(tmp);
> + tmp = "";
> + }
> + } else {
> + tmp += s[i];
> + }
> + }
> +
> + if ( tmp.size() ) {
> + r.push_back(tmp);
> + }
> +
> + return r;
> + }
> +
> + enum ParserState {
> + kParseStart,
> + kParseCompressibleContentType,
> + kParseRemoveAcceptEncoding,
> + kParseEnable,
> + kParseCache,
> + kParseDisallow,
> + };
> +
> + void Configuration::AddHostConfiguration(HostConfiguration * hc){
> + host_configurations_.push_back(hc);
> + }
> +
> + void HostConfiguration::add_disallow(const std::string & disallow) {
> + disallows_.push_back(disallow);
> + }
> +
> + void HostConfiguration::add_compressible_content_type(const std::string & content_type) {
> + compressible_content_types_.push_back(content_type);
> + }
> +
> + HostConfiguration * Configuration::Find(const char * host, int host_length) {
> + HostConfiguration * host_configuration = host_configurations_[0];
> +
> + std::string shost(host, host_length);
> +
> + for (size_t i = 1; i < host_configurations_.size(); i++ ) {
> + if (host_configurations_[i]->host() == shost){
> + host_configuration = host_configurations_[i];
> + break;
> + }
> + }
> +
> + return host_configuration;
> + }
> +
> + bool HostConfiguration::IsUrlAllowed(const char * url, int url_len) {
> + string surl(url, url_len);
> +
> + for (size_t i = 0; i < disallows_.size(); i++) {
> + if ( fnmatch (disallows_[i].c_str(), surl.c_str(), 0) == 0 ) {
> + info("url [%s] disabled for compression, matched on pattern [%s]",
> + surl.c_str(), disallows_[i].c_str());
> + return false;
> + }
> + }
> +
> + return true;
> + }
> +
> + bool HostConfiguration::ContentTypeIsCompressible(const char * content_type, int content_type_length) {
> + string scontent_type(content_type, content_type_length);
> + bool is_match = false;
> +
> + for (size_t i = 0; i < compressible_content_types_.size(); i++) {
> + const char* match_string = compressible_content_types_[i].c_str();
> + bool exclude = match_string[0] == '!';
> + if (exclude) {
> + match_string++;//skip '!'
> + }
> + if ( fnmatch (match_string, scontent_type.c_str(), 0) == 0 ) {
> + info("compressible content type [%s], matched on pattern [%s]",
> + scontent_type.c_str(), compressible_content_types_[i].c_str());
> + is_match = !exclude;
> + }
> + }
> +
> + return is_match;
> + }
> +
> + Configuration * Configuration::Parse(const char * path ) {
> + string pathstring(path);
> +
> + // If we have a path and it's not an absolute path, make it relative to the
> + // configuration directory.
> + if (!pathstring.empty() && pathstring[0] != '/') {
> + pathstring.assign(TSConfigDirGet());
> + pathstring.append("/");
> + pathstring.append(path);
> + }
> +
> + trim_if(pathstring, isspace);
> +
> + Configuration * c = new Configuration();
> + HostConfiguration * current_host_configuration = new HostConfiguration("");
> + c->AddHostConfiguration(current_host_configuration);
> + current_host_configuration->add_compressible_content_type("text/*");
> + current_host_configuration->add_compressible_content_type("application/xml*");
> + current_host_configuration->add_compressible_content_type("*javascript*");
> + current_host_configuration->add_compressible_content_type("image/svg+xml");
> +
> +
> + if (pathstring.empty()) {
> + return c;
> + }
> +
> + path = pathstring.c_str();
> + info("Parsing file \"%s\"", path);
> + std::ifstream f;
> +
> + size_t lineno = 0;
> +
> + f.open(path, std::ios::in);
> +
> + if (!f.is_open()) {
> + warning("could not open file [%s], skip",path);
> + return c;
> + }
> +
> + enum ParserState state = kParseStart;
> +
> + while (!f.eof()) {
> + std::string line;
> + getline(f, line);
> + ++lineno;
> +
> + trim_if(line, isspace);
> + if (line.size() == 0) {
> + continue;
> + }
> +
> + vector<string> v = tokenize( line, isspace );
> +
> + for(size_t i = 0; i < v.size(); i++ ) {
> + string token = v[i];
> + trim_if(token, isspace);
> +
> + //should not happen
> + if (!token.size()) continue;
> +
> + //once a comment is encountered, we are done processing the line
> + if (token[0] == '#') break;
> +
> + switch(state) {
> + case kParseStart:
> + if ( (token[0] == '[') && (token[token.size()-1] == ']')){
> + std::string current_host = token.substr(1,token.size()-2);
> + current_host_configuration = new HostConfiguration(current_host);
> + c->AddHostConfiguration(current_host_configuration);
> + } else if (token == "compressible-content-type" ) {
> + state = kParseCompressibleContentType;
> + } else if (token == "remove-accept-encoding" ) {
> + state = kParseRemoveAcceptEncoding;
> + } else if (token == "enabled" ) {
> + state = kParseEnable;
> + } else if (token == "cache" ) {
> + state = kParseCache;
> + } else if (token == "disallow" ) {
> + state = kParseDisallow;
> + }
> + else {
> + warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno);
> + }
> + break;
> + case kParseCompressibleContentType:
> + current_host_configuration->add_compressible_content_type(token);
> + state = kParseStart;
> + break;
> + case kParseRemoveAcceptEncoding:
> + current_host_configuration->set_remove_accept_encoding(token == "true");
> + state = kParseStart;
> + break;
> + case kParseEnable:
> + current_host_configuration->set_enabled(token == "true");
> + state = kParseStart;
> + break;
> + case kParseCache:
> + current_host_configuration->set_cache(token == "true");
> + state = kParseStart;
> + break;
> + case kParseDisallow:
> + current_host_configuration->add_disallow(token);
> + state = kParseStart;
> + break;
> + }
> + }
> + }
> +
> + if (state != kParseStart) {
> + warning("the parser state indicates that data was expected when it reached the end of the file (%d)", state);
> + }
> +
> + return c;
> + } //Configuration::Parse
> +} //namespace
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/configuration.h b/plugins/experimental/ats_speed/gzip/configuration.h
> new file mode 100644
> index 0000000..b38cb64
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/configuration.h
> @@ -0,0 +1,84 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +
> +#ifndef GZIP_CONFIGURATION_H_
> +#define GZIP_CONFIGURATION_H_
> +
> +#include <string>
> +#include <vector>
> +#include "debug_macros.h"
> +
> +namespace Gzip {
> + class HostConfiguration {
> + public: //todo -> only configuration should be able to construct hostconfig
> + explicit HostConfiguration(const std::string & host)
> + : host_(host)
> + , enabled_(true)
> + , cache_(true)
> + , remove_accept_encoding_(false)
> + {}
> +
> + inline bool enabled() { return enabled_; }
> + inline void set_enabled(bool x) { enabled_ = x; }
> + inline bool cache() { return cache_; }
> + inline void set_cache(bool x) { cache_ = x; }
> + inline bool remove_accept_encoding() { return remove_accept_encoding_; }
> + inline void set_remove_accept_encoding(bool x) { remove_accept_encoding_ = x; }
> + inline std::string host() { return host_; }
> + void add_disallow(const std::string & disallow);
> + void add_compressible_content_type(const std::string & content_type);
> + bool IsUrlAllowed(const char * url, int url_len);
> + bool ContentTypeIsCompressible(const char * content_type, int content_type_length);
> +
> + private:
> + std::string host_;
> + bool enabled_;
> + bool cache_;
> + bool remove_accept_encoding_;
> + std::vector<std::string> compressible_content_types_;
> + std::vector<std::string> disallows_;
> + DISALLOW_COPY_AND_ASSIGN(HostConfiguration);
> + };//class HostConfiguration
> +
> + class Configuration {
> + friend class HostConfiguration;
> + public:
> + static Configuration * Parse(const char * path);
> + HostConfiguration * Find(const char * host, int host_length);
> + inline HostConfiguration * GlobalConfiguration() {
> + return host_configurations_[0];
> + }
> +
> + private:
> + explicit Configuration() {}
> + void AddHostConfiguration(HostConfiguration * hc);
> +
> + std::vector<HostConfiguration *> host_configurations_;
> + //todo: destructor. delete owned host configurations
> + DISALLOW_COPY_AND_ASSIGN(Configuration);
> + }; //class Configuration
> +
> +}//namespace
> +
> +#endif
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/debug_macros.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/debug_macros.h b/plugins/experimental/ats_speed/gzip/debug_macros.h
> new file mode 100644
> index 0000000..151de31
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/debug_macros.h
> @@ -0,0 +1,59 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#ifndef _DBG_MACROS_H
> +#define _DBG_MACROS_H
> +
> +#include <ts/ts.h>
> +
> +#define TAG "gzip"
> +
> +#define debug(fmt, args...) do { \
> + TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + } while (0)
> +
> +#define info(fmt, args...) do { \
> + TSDebug(TAG, "INFO: " fmt, ##args ); \
> + } while (0)
> +
> +#define warning(fmt, args...) do { \
> + TSDebug(TAG, "WARNING: " fmt, ##args ); \
> +} while (0)
> +
> +#define error(fmt, args...) do { \
> + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> +} while (0)
> +
> +#define fatal(fmt, args...) do { \
> + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + exit(-1); \
> +} while (0)
> +
> +//FIXME: this one doesn't deserve to be here
> +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
> + TypeName(const TypeName&); \
> + void operator=(const TypeName&)
> +
> +#endif //_DBG_MACROS_H
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/gzip.cc b/plugins/experimental/ats_speed/gzip/gzip.cc
> new file mode 100644
> index 0000000..1397762
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/gzip.cc
> @@ -0,0 +1,826 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +#define __STDC_LIMIT_MACROS
> +#define __STDC_CONSTANT_MACROS
> +#include <stdint.h>
> +
> +#include <string>
> +#include <string.h>
> +#include <zlib.h>
> +#include <ts/ts.h>
> +#include "debug_macros.h"
> +#include "misc.h"
> +#include "configuration.h"
> +//#include "ink_defs.h"
> +
> +using namespace std;
> +using namespace Gzip;
> +
> +//FIXME: custom dictionaries would be nice. configurable/content-type?
> +//FIXME: look into autoscaling the compression level based on connection speed
> +// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
> +// might be served better with little or no compression at all
> +//FIXME: look into compressing from the task thread pool
> +//FIXME: make normalizing accept encoding configurable
> +
> +// from mod_deflate:
> +// ZLIB's compression algorithm uses a
> +// 0-9 based scale that GZIP does where '1' is 'Best speed'
> +// and '9' is 'Best compression'. Testing has proved level '6'
> +// to be about the best level to use in an HTTP Server.
> +
> +const int ZLIB_COMPRESSION_LEVEL = 6;
> +
> +int arg_idx_hooked;
> +int arg_idx_host_configuration;
> +int arg_idx_url_disallowed;
> +
> +
> +const char * global_hidden_header_name;
> +Configuration* config = NULL;
> +const char *dictionary = NULL;
> +
> +static GzipData *
> +gzip_data_alloc(int compression_type)
> +{
> + GzipData *data;
> + int err;
> +
> + data = (GzipData *) TSmalloc(sizeof(GzipData));
> + data->downstream_vio = NULL;
> + data->downstream_buffer = NULL;
> + data->downstream_reader = NULL;
> + data->downstream_length = 0;
> + data->state = transform_state_initialized;
> + data->compression_type = compression_type;
> + data->zstrm.next_in = Z_NULL;
> + data->zstrm.avail_in = 0;
> + data->zstrm.total_in = 0;
> + data->zstrm.next_out = Z_NULL;
> + data->zstrm.avail_out = 0;
> + data->zstrm.total_out = 0;
> + data->zstrm.zalloc = gzip_alloc;
> + data->zstrm.zfree = gzip_free;
> + data->zstrm.opaque = (voidpf) 0;
> + data->zstrm.data_type = Z_ASCII;
> +
> + int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
> +
> + err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
> +
> + if (err != Z_OK) {
> + fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
> + }
> +
> + if (dictionary) {
> + err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
> + if (err != Z_OK) {
> + fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
> + }
> + }
> +
> + return data;
> +}
> +
> +
> +static void
> +gzip_data_destroy(GzipData * data)
> +{
> + TSReleaseAssert(data);
> +
> + //deflateEnd returnvalue ignore is intentional
> + //it would spew log on every client abort
> + deflateEnd(&data->zstrm);
> +
> + if (data->downstream_buffer) {
> + TSIOBufferDestroy(data->downstream_buffer);
> + }
> +
> + TSfree(data);
> +}
> +
> +static TSReturnCode
> +gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type)
> +{
> + TSReturnCode ret;
> + TSMLoc ce_loc;
> +
> + // Delete Content-Encoding if present???
> +
> + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Content-Encoding", sizeof("Content-Encoding") - 1, &ce_loc)) == TS_SUCCESS) {
> + if (compression_type == COMPRESSION_TYPE_DEFLATE) {
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", sizeof("deflate") - 1);
> + } else if (compression_type == COMPRESSION_TYPE_GZIP) {
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", sizeof("gzip") - 1);
> + }
> + if (ret == TS_SUCCESS) {
> + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot add the Content-Encoding header");
> + }
> +
> + return ret;
> +}
> +
> +static TSReturnCode
> +gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
> +{
> + TSReturnCode ret;
> + TSMLoc ce_loc;
> +
> + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Vary", sizeof("Vary") - 1);
> + if (ce_loc) {
> + int idx, count, len;
> + const char *value;
> +
> + count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc);
> + for(idx=0; idx<count; idx++) {
> + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len);
> + if (len &&
> + strncasecmp("Accept-Encoding", value, len) == 0) {
> + // Bail, Vary: Accept-Encoding already sent from origin
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + return TS_SUCCESS;
> + }
> + }
> +
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1);
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + } else {
> + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Vary", sizeof("Vary") - 1, &ce_loc)) == TS_SUCCESS) {
> + if ((ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1)) == TS_SUCCESS) {
> + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
> + }
> +
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot add/update the Vary header");
> + }
> +
> + return ret;
> +}
> +
> +//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
> +// specify a very header..
> +static TSReturnCode
> +gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
> +{
> + TSReturnCode ret = TS_SUCCESS;
> + TSMLoc ce_loc;
> +
> + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
> +
> + if (ce_loc) {
> + int changetag = 1;
> + int strl;
> + const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
> + //do not alter weak etags.
> + //FIXME: consider just making the etag weak for compressed content
> + if (strl >= 2) {
> + if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
> + changetag = 0;
> + }
> + if (changetag) {
> + ret = TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
> + }
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot handle the %s header", TS_MIME_FIELD_ETAG);
> + }
> +
> + return ret;
> +}
> +
> +//FIXME: some things are potentially compressible. those responses
> +static void
> +gzip_transform_init(TSCont contp, GzipData * data)
> +{
> + //update the vary, content-encoding, and etag response headers
> + //prepare the downstream for transforming
> +
> + TSVConn downstream_conn;
> + TSMBuffer bufp;
> + TSMLoc hdr_loc;
> +
> + data->state = transform_state_output;
> +
> + if (TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc) != TS_SUCCESS) {
> + error("Error TSHttpTxnTransformRespGet");
> + return;
> + }
> +
> + if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == TS_SUCCESS &&
> + gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS &&
> + gzip_etag_header(bufp, hdr_loc) == TS_SUCCESS) {
> + downstream_conn = TSTransformOutputVConnGet(contp);
> + data->downstream_buffer = TSIOBufferCreate();
> + data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
> + data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
> + }
> +
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> +}
> +
> +
> +
> +static void
> +gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
> +{
> + TSIOBufferBlock downstream_blkp;
> + const char *upstream_buffer;
> + char *downstream_buffer;
> + int64_t upstream_length, downstream_length;
> + int err;
> +
> + while (amount > 0) {
> + downstream_blkp = TSIOBufferReaderStart(upstream_reader);
> + if (!downstream_blkp) {
> + error("couldn't get from IOBufferBlock");
> + return;
> + }
> +
> + upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
> + if (!upstream_buffer) {
> + error("couldn't get from TSIOBufferBlockReadStart");
> + return;
> + }
> +
> + if (upstream_length > amount) {
> + upstream_length = amount;
> + }
> +
> + data->zstrm.next_in = (unsigned char *) upstream_buffer;
> + data->zstrm.avail_in = upstream_length;
> +
> + while (data->zstrm.avail_in > 0) {
> + downstream_blkp = TSIOBufferStart(data->downstream_buffer);
> + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
> +
> + data->zstrm.next_out = (unsigned char *) downstream_buffer;
> + data->zstrm.avail_out = downstream_length;
> +
> + err = deflate(&data->zstrm, Z_NO_FLUSH);
> +
> + if (err != Z_OK)
> + warning("deflate() call failed: %d", err);
> +
> + if (downstream_length > data->zstrm.avail_out) {
> + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
> + data->downstream_length += (downstream_length - data->zstrm.avail_out);
> + }
> +
> + if (data->zstrm.avail_out > 0) {
> + if (data->zstrm.avail_in != 0) {
> + error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
> + }
> + }
> + }
> +
> + TSIOBufferReaderConsume(upstream_reader, upstream_length);
> + amount -= upstream_length;
> + }
> +}
> +
> +static void
> +gzip_transform_finish(GzipData * data)
> +{
> + if (data->state == transform_state_output) {
> + TSIOBufferBlock downstream_blkp;
> + char *downstream_buffer;
> + int64_t downstream_length;
> + int err;
> +
> + data->state = transform_state_finished;
> +
> + for (;;) {
> + downstream_blkp = TSIOBufferStart(data->downstream_buffer);
> +
> + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
> + data->zstrm.next_out = (unsigned char *) downstream_buffer;
> + data->zstrm.avail_out = downstream_length;
> +
> + err = deflate(&data->zstrm, Z_FINISH);
> +
> + if (downstream_length > (int64_t) data->zstrm.avail_out) {
> + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
> + data->downstream_length += (downstream_length - data->zstrm.avail_out);
> + }
> +
> + if (err == Z_OK) { /* some more data to encode */
> + continue;
> + }
> +
> + if (err != Z_STREAM_END) {
> + warning("deflate should report Z_STREAM_END");
> + }
> + break;
> + }
> +
> + if (data->downstream_length != (int64_t) (data->zstrm.total_out)) {
> + error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
> + data->zstrm.total_out);
> + }
> +
> + gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
> + }
> +}
> +
> +
> +static void
> +gzip_transform_do(TSCont contp)
> +{
> + TSVIO upstream_vio;
> + GzipData *data;
> + int64_t upstream_todo;
> + int64_t upstream_avail;
> + int64_t downstream_bytes_written;
> +
> + data = (GzipData*)TSContDataGet(contp);
> + if (data->state == transform_state_initialized) {
> + gzip_transform_init(contp, data);
> + }
> +
> + upstream_vio = TSVConnWriteVIOGet(contp);
> + downstream_bytes_written = data->downstream_length;
> +
> + if (!TSVIOBufferGet(upstream_vio)) {
> + gzip_transform_finish(data);
> +
> + TSVIONBytesSet(data->downstream_vio, data->downstream_length);
> +
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> + return;
> + }
> +
> + upstream_todo = TSVIONTodoGet(upstream_vio);
> +
> + if (upstream_todo > 0) {
> + upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
> +
> + if (upstream_todo > upstream_avail) {
> + upstream_todo = upstream_avail;
> + }
> +
> + if (upstream_todo > 0) {
> + gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
> + TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
> + }
> + }
> +
> + if (TSVIONTodoGet(upstream_vio) > 0) {
> + if (upstream_todo > 0) {
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
> + }
> + } else {
> + gzip_transform_finish(data);
> + TSVIONBytesSet(data->downstream_vio, data->downstream_length);
> +
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> +
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
> + }
> +}
> +
> +
> +static int
> +gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
> +{
> + if (TSVConnClosedGet(contp)) {
> + gzip_data_destroy((GzipData*)TSContDataGet(contp));
> + TSContDestroy(contp);
> + return 0;
> + } else {
> + switch (event) {
> + case TS_EVENT_ERROR:{
> + debug("gzip_transform: TS_EVENT_ERROR starts");
> + TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
> + }
> + break;
> + case TS_EVENT_VCONN_WRITE_COMPLETE:
> + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
> + break;
> + case TS_EVENT_VCONN_WRITE_READY:
> + gzip_transform_do(contp);
> + break;
> + case TS_EVENT_IMMEDIATE:
> + gzip_transform_do(contp);
> + break;
> + default:
> + warning("unknown event [%d]", event);
> + gzip_transform_do(contp);
> + break;
> + }
> + }
> +
> + return 0;
> +}
> +
> +
> +static int
> +gzip_transformable(TSHttpTxn txnp, int server, HostConfiguration * host_configuration, int *compress_type)
> +{
> + /* Server response header */
> + TSMBuffer bufp;
> + TSMLoc hdr_loc;
> + TSMLoc field_loc;
> +
> + /* Client request header */
> + TSMBuffer cbuf;
> + TSMLoc chdr;
> + TSMLoc cfield;
> +
> + const char *value;
> + int nvalues;
> + int i, compression_acceptable, len;
> +
> + TSHttpStatus resp_status;
> + if (server) {
> + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
> + } else {
> + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
> + }
> + resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> +
> + //conservatively pick some statusses to compress
> + if (!(resp_status == 200 || resp_status == 404 || resp_status == 500)) {
> + info("http response status [%d] is not compressible", resp_status);
> + return 0;
> + }
> +
> + TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
> +
> + //the only compressible method is currently GET.
> + int method_length;
> + const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length);
> + if (!(method_length == TS_HTTP_LEN_GET && memcmp(method, TS_HTTP_METHOD_GET, TS_HTTP_LEN_GET) == 0)) {
> + debug("method is not GET, not compressible");
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> + return 0;
> + }
> +
> + cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + if (cfield != TS_NULL_MLOC) {
> + compression_acceptable = 0;
> + nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
> + for (i=0; i<nvalues; i++) {
> + value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len);
> + if (!value) {
> + continue;
> + }
> +
> + if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
> + compression_acceptable = 1;
> + *compress_type = COMPRESSION_TYPE_DEFLATE;
> + break;
> + } else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) {
> + compression_acceptable = 1;
> + *compress_type = COMPRESSION_TYPE_GZIP;
> + break;
> + }
> + }
> +
> + TSHandleMLocRelease(cbuf, chdr, cfield);
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> +
> + if (!compression_acceptable) {
> + info("no acceptable encoding found in request header, not compressible");
> + return 0;
> + }
> + } else {
> + info("no acceptable encoding found in request header, not compressible");
> + TSHandleMLocRelease(cbuf, chdr, cfield);
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> + return 0;
> + }
> +
> + if (server) {
> + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
> + } else {
> + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
> + }
> +
> + /* If there already exists a content encoding then we don't want
> + to do anything. */
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
> + if (field_loc) {
> + info("response is already content encoded, not compressible");
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return 0;
> + }
> +
> + /* We only want to do gzip compression on documents that have a
> + content type of "text/" or "application/x-javascript". */
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
> + if (!field_loc) {
> + info("no content type header found, not compressible");
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return 0;
> + }
> +
> + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len);
> +
> + int rv = host_configuration->ContentTypeIsCompressible(value, len);
> + if (!rv) {
> + info("content-type [%.*s] not compressible", len, value);
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return rv;
> +}
> +
> +
> +static void
> +gzip_transform_add(TSHttpTxn txnp, int /* server ATS_UNUSED */, HostConfiguration * hc, int compress_type)
> +{
> + int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked);
> + if (tmp) {
> + //happens on cache_stale_hit
> + debug("transform hook already set, bail");
> + return;
> + } else {
> + TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &GZIP_ONE);
> + info("adding compression transform");
> + }
> +
> + TSHttpTxnUntransformedRespCache(txnp, 1);
> +
> + if (!hc->cache()) {
> + debug("@@@@@ Gzip cache disabled");
> + TSHttpTxnTransformedRespCache(txnp, 0);
> + } else {
> + TSMBuffer bufp;
> + TSMLoc hdr_loc, field_loc;
> + int cache = 1;
> + if (TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS || TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) {
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "@gzip_nocache", strlen("@gzip_nocache"));
> + if (field_loc) {
> + cache = 0;
> + debug("@@@@@ Gzip disallows cacheing of transformed response");
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + } else {
> + debug("@@@@ Gzip allows cacheing of transformed response");
> + }
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + }
> +
> + TSHttpTxnTransformedRespCache(txnp, cache);
> + }
> +
> + TSVConn connp;
> + GzipData *data;
> +
> + connp = TSTransformCreate(gzip_transform, txnp);
> + data = gzip_data_alloc(compress_type);
> + data->txn = txnp;
> +
> + TSContDataSet(connp, data);
> + TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
> +}
> +
> +static int
> +cache_transformable(TSHttpTxn txnp)
> +{
> + int obj_status;
> +
> + if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
> + warning("Couldn't get cache status of object");
> + return 0;
> + }
> + if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) {
> + info("stale cache hit");
> + return 0;
> + }
> + if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
> + info("fresh cache hit");
> + return 1;
> + }
> +
> + return 0;
> +}
> +
> +HostConfiguration *
> +find_host_configuration(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer bufp, TSMLoc locp)
> +{
> + TSMLoc fieldp = TSMimeHdrFieldFind(bufp, locp, TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST);
> +
> + if (fieldp) {
> + int strl;
> + const char *strv = TSMimeHdrFieldValueStringGet(bufp, locp, fieldp, -1, &strl);
> + TSHandleMLocRelease(bufp, locp, fieldp);
> +
> + HostConfiguration * host_configuration = config->Find(strv, strl);
> + return host_configuration;
> + }
> +
> + return config->GlobalConfiguration();
> +}
> +
> +
> +static int
> +transform_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata)
> +{
> + TSHttpTxn txnp = (TSHttpTxn) edata;
> + int compress_type = COMPRESSION_TYPE_DEFLATE;
> +
> + switch (event) {
> + case TS_EVENT_HTTP_READ_REQUEST_HDR:
> + {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + int url_len;
> + char * url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
> + HostConfiguration * hc = find_host_configuration(txnp, req_buf, req_loc);
> + //we could clone the hosting configuration here, to make it deletable on reload?
> + TSHttpTxnArgSet(txnp, arg_idx_host_configuration, (void *) hc);
> +
> + if (!hc->enabled() || !hc->IsUrlAllowed(url, url_len)) {
> + //FIXME: no double negatives
> + TSHttpTxnArgSet(txnp, arg_idx_url_disallowed, (void *) &GZIP_ONE);
> + info("url [%.*s] not allowed", url_len, url);
> + } else {
> + normalize_accept_encoding(txnp, req_buf, req_loc);
> + }
> + TSfree(url);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_READ_RESPONSE_HDR:
> + {
> + //os: the accept encoding header needs to be restored..
> + //otherwise the next request won't get a cache hit on this
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if (hc != NULL) {
> + if (hc->remove_accept_encoding()) {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + restore_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + }
> +
> + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
> + if ( allowed && gzip_transformable(txnp, 1, hc, &compress_type)) {
> + gzip_transform_add(txnp, 1, hc, compress_type);
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_SEND_REQUEST_HDR:
> + {
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if (hc!=NULL) {
> + if (hc->remove_accept_encoding()) {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + hide_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
> + {
> + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if ( hc != NULL ) {
> + if (allowed && cache_transformable(txnp) && gzip_transformable(txnp, 0, hc, &compress_type)) {
> + gzip_transform_add(txnp, 0, hc, compress_type);
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + default:
> + fatal("gzip transform unknown event");
> + }
> +
> + return 0;
> +}
> +
> +
> +static void
> +read_configuration(TSCont contp) {
> + const char * path = (const char *)TSContDataGet(contp);
> + Configuration * newconfig = Configuration::Parse(path);
> +
> + Configuration * oldconfig =__sync_lock_test_and_set(&config, newconfig);
> + debug("config swapped,old config %p", oldconfig);
> +
> + //FIXME: we have leaked.
> + //consider cloning or refcounting the configuration passed to the txn
> + //to make deleting the old configuration possible
> + //if (config != NULL )
> + // delete config;
> +}
> +
> +static int
> +management_update(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
> +{
> + TSReleaseAssert(event == TS_EVENT_MGMT_UPDATE);
> + info("management update event received");
> + read_configuration(contp);
> + return 0;
> +}
> +
> +
> +void
> +TSPluginInit(int argc, const char *argv[])
> +{
> + string config_path;
> +
> + if (argc > 2) {
> + fatal("the gzip plugin does not accept more than 1 plugin argument");
> + } else if (argc == 2) {
> + config_path = std::string(argv[1]);
> + }
> +
> + info("TSPluginInit %s", argv[0]);
> +
> + if (!register_plugin()) {
> + fatal("The gzip plugin failed to register");
> + }
> +
> + //if (argc == 2) {
> + // dictionary = load_dictionary(argv[1]);
> + //}
> +
> + if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", &arg_idx_hooked) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> + if (TSHttpArgIndexReserve("gzip", "for storing if compression is applicable", &arg_idx_host_configuration) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> + if (TSHttpArgIndexReserve("gzip", "for storing if compression is disallowed for this txn", &arg_idx_url_disallowed) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> +
> + global_hidden_header_name = init_hidden_header_name();
> +
> + TSCont management_contp = TSContCreate(management_update, NULL);
> + //fixme: never freed. there is no shutdown event?
> + char * p = (char*)TSmalloc(config_path.size()+1);
> + strcpy(p,config_path.c_str());
> + TSContDataSet(management_contp,(void*)p);
> + TSMgmtUpdateRegister(management_contp, TAG);
> + read_configuration(management_contp);
> +
> + TSCont transform_contp = TSContCreate(transform_plugin, NULL);
> + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp);
> +
> + info("loaded");
> +}
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.config
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/gzip.config b/plugins/experimental/ats_speed/gzip/gzip.config
> new file mode 100644
> index 0000000..81e9fa4
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/gzip.config
> @@ -0,0 +1,6 @@
> +# Set some global options first
> +cache true
> +enabled true
> +remove-accept-encoding false
> +compressible-content-type text/*
> +compressible-content-type *javascript*
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/misc.cc b/plugins/experimental/ats_speed/gzip/misc.cc
> new file mode 100644
> index 0000000..0ea6911
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/misc.cc
> @@ -0,0 +1,197 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#include "ts/ts.h"
> +//#include "ink_defs.h"
> +#include <stdint.h>
> +#include <inttypes.h>
> +#include "misc.h"
> +#include <string.h>
> +#include "debug_macros.h"
> +
> +voidpf
> +gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size)
> +{
> + return (voidpf) TSmalloc(items * size);
> +}
> +
> +void
> +gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address)
> +{
> + TSfree(address);
> +}
> +
> +void
> +normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + int deflate = 0;
> + int gzip = 0;
> +
> + //remove the accept encoding field(s),
> + //while finding out if gzip or deflate is supported.
> + while (field) {
> + TSMLoc tmp;
> +
> + if (!deflate && !gzip) {
> + int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field);
> +
> + while (value_count > 0) {
> + int val_len = 0;
> + const char *val;
> +
> + --value_count;
> + val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
> +
> + if (val_len == (int) strlen("gzip"))
> + gzip = !strncmp(val, "gzip", val_len);
> + else if (val_len == (int) strlen("deflate"))
> + deflate = !strncmp(val, "deflate", val_len);
> + }
> + }
> +
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +
> + //append a new accept-encoding field in the header
> + if (deflate || gzip) {
> + TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> +
> + if (gzip) {
> + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
> + info("normalized accept encoding to gzip");
> + } else if (deflate) {
> + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", strlen("deflate"));
> + info("normalized accept encoding to deflate");
> + }
> +
> + TSMimeHdrFieldAppend(reqp, hdr_loc, field);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + }
> +}
> +
> +void
> +hide_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + while (field) {
> + TSMLoc tmp;
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +}
> +
> +void
> +restore_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
> +
> + while (field) {
> + TSMLoc tmp;
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +}
> +
> +const char *
> +init_hidden_header_name()
> +{
> + char * hidden_header_name;
> + const char *var_name = "proxy.config.proxy_name";
> + TSMgmtString result;
> +
> + if (TSMgmtStringGet(var_name, &result) != TS_SUCCESS) {
> + fatal("failed to get server name");
> + } else {
> + int hidden_header_name_len = strlen("x-accept-encoding-") + strlen(result);
> + hidden_header_name = (char *) TSmalloc(hidden_header_name_len + 1);
> + hidden_header_name[hidden_header_name_len] = 0;
> + sprintf(hidden_header_name, "x-accept-encoding-%s", result);
> + }
> + return hidden_header_name;
> +}
> +
> +int
> +register_plugin()
> +{
> + TSPluginRegistrationInfo info;
> +
> + info.plugin_name = (char*)"gzip";
> + info.vendor_name = (char*)"Apache";
> + info.support_email = (char*)"dev@trafficserver.apache.org";
> +
> + if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
> + return 0;
> + }
> + return 1;
> +}
> +
> +const char *
> +load_dictionary(const char *preload_file)
> +{
> + char *dict = (char *) malloc(800000);
> + uLong dictId = adler32(0L, Z_NULL, 0);
> + uLong *adler = &dictId;
> +
> + FILE *fp;
> + int i = 0;
> +
> + fp = fopen(preload_file, "r");
> + if (!fp) {
> + fatal("gzip-transform: ERROR: Unable to open dict file %s", preload_file);
> + }
> +
> + /* dict = (char *) calloc(8000, sizeof(char)); */
> +
> + i = 0;
> + while (!feof(fp)) {
> + if (fscanf(fp, "%s\n", dict + i) == 1) {
> + i = strlen(dict);
> + strcat(dict + i, " ");
> + ++i;
> + }
> + }
> + dict[i - 1] = '\0';
> +
> + /* TODO get the adler compute right */
> + *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
> + return dict;
> +}
> +
> +void
> +gzip_log_ratio(int64_t in, int64_t out)
> +{
> + // if (in) {
> + // info("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, ((float) (in - out) / in));
> + //} else {
> + // debug("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, 0.0F);
> + // }
> +}
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/misc.h b/plugins/experimental/ats_speed/gzip/misc.h
> new file mode 100644
> index 0000000..c44fb6b
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/misc.h
> @@ -0,0 +1,84 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#ifndef _GZIP_MISC_H_
> +#define _GZIP_MISC_H_
> +
> +#define __STDC_LIMIT_MACROS
> +#define __STDC_CONSTANT_MACROS
> +#include <stdint.h>
> +#include <zlib.h>
> +#include <ts/ts.h>
> +#include <stdlib.h> //exit()
> +#include <stdio.h>
> +
> +//zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html
> +static const int ZLIB_MEMLEVEL = 9; //min=1 (optimize for memory),max=9 (optimized for speed)
> +static const int WINDOW_BITS_DEFLATE = -15;
> +static const int WINDOW_BITS_GZIP = 31;
> +
> +//misc
> +static const int COMPRESSION_TYPE_DEFLATE = 1;
> +static const int COMPRESSION_TYPE_GZIP = 2;
> +//this one is just for txnargset/get to point to
> +static const int GZIP_ONE = 1;
> +static const int DICT_PATH_MAX = 512;
> +static const int DICT_ENTRY_MAX = 2048;
> +
> +//this one is used to rename the accept encoding header
> +//it will be restored later on
> +//to make it work, the name must be different then downstream proxies though
> +//otherwise the downstream will restore the accept encoding header
> +
> +enum transform_state
> +{
> + transform_state_initialized,
> + transform_state_output,
> + transform_state_finished
> +};
> +
> +typedef struct
> +{
> + TSHttpTxn txn;
> + TSVIO downstream_vio;
> + TSIOBuffer downstream_buffer;
> + TSIOBufferReader downstream_reader;
> + int downstream_length;
> + z_stream zstrm;
> + enum transform_state state;
> + int compression_type;
> +} GzipData;
> +
> +
> +voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
> +void gzip_free(voidpf opaque, voidpf address);
> +void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
> +void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
> +void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
> +const char * init_hidden_header_name();
> +int check_ts_version();
> +int register_plugin();
> +const char *load_dictionary(const char *preload_file);
> +void gzip_log_ratio(int64_t in, int64_t out);
> +
> +#endif
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/scripts/prepare_psol.sh b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> new file mode 100755
> index 0000000..4862249
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> @@ -0,0 +1,95 @@
> +#!/bin/bash
> +#
> +# Copyright 2012 Google Inc.
> +#
> +# Licensed under the Apache License, Version 2.0 (the "License");
> +# you may not use this file except in compliance with the License.
> +# You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +#
> +# Author: jefftk@google.com (Jeff Kaufman)
> +# Author: oschaaf@we-amp.com (Otto van der Schaaf)
> +#
> +# Usage:
> +# scripts/prepare_psol.sh /path/to/mod_pagespeed/src
> +#
> +# Creates a directory psol/ and copies headers and a few source files from a
> +# depot_tools (glient) checkout into psol/include. Along with creating
> +# binaries, this is a step in preparing psol.tar.gz for distribution.
> +#
> +
> +set -u # check for undefined variables
> +set -e # exit on failed commands
> +
> +if [ "$(basename "$PWD")" != "ats_speed" ] ; then
> + echo "$(basename $0) must be invoked from the ats_speed directory"
> + exit 1
> +fi
> +
> +if [ $# -ne 1 ] ; then
> + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
> + exit 1
> +fi
> +
> +MOD_PAGESPEED_SRC="$1"
> +
> +if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
> + basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
> + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
> + exit 1
> +fi
> +
> +if [ -e psol ] ; then
> + echo "A psol/ directory already exists. Move it somewhere else and rerun."
> + exit 1
> +fi
> +mkdir psol/
> +# Copy over the .h files, plus a few selected .cc and .c files.
> +rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
> + --exclude=".svn" \
> + --exclude=".git" \
> + --include='*.h' \
> + --include='*/' \
> + --include="apr_thread_compatible_pool.cc" \
> + --include="serf_url_async_fetcher.cc" \
> + --include="apr_mem_cache.cc" \
> + --include="key_value_codec.cc" \
> + --include="apr_memcache2.c" \
> + --include="loopback_route_fetcher.cc" \
> + --include="add_headers_fetcher.cc" \
> + --include="console_css_out.cc" \
> + --include="console_out.cc" \
> + --include="dense_hash_map" \
> + --include="dense_hash_set" \
> + --include="sparse_hash_map" \
> + --include="sparse_hash_set" \
> + --include="sparsetable" \
> + --include="mod_pagespeed_console_out.cc" \
> + --include="mod_pagespeed_console_css_out.cc" \
> + --include="mod_pagespeed_console_html_out.cc" \
> + --exclude='*'
> +mkdir -p psol/lib/Debug/linux/ia32
> +mkdir -p psol/lib/Debug/linux/x64
> +mkdir -p psol/lib/Release/linux/ia32
> +mkdir -p psol/lib/Release/linux/x64
> +
> +# Log that we did this.
> +SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
> +SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL | awk -F/ '{print $(NF-1)}')"
> +
> +DATE="$(date +%F)"
> +echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
> + >> psol/include_history.txt
> +
> +echo
> +echo "Output is in psol/include. Now put binaries in psol/lib following"
> +echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
> +echo "and then you can distribute PSOL."
> +
>
[3/3] git commit: ats_speed: PageSpeed optimization plugin
Posted by os...@apache.org.
ats_speed: PageSpeed optimization plugin
Pull in the plugin code from https://github.com/We-Amp/ats_speed.
This adds the plugin as-is, and doesn't wire up to automake yet.
As this plugin needs to download external libraries to
build, a custom make file is added.
See README.md for setting this up.
For more information see
https://developers.google.com/speed/pagespeed/optimization and
http://www.atsspeed.com/
Plugin is donated by We-Amp, ip cleared via:
http://incubator.apache.org/ip-clearance/ats-ats_speed.html
Jira ticket: https://issues.apache.org/jira/browse/TS-2926
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/083abd4f
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/083abd4f
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/083abd4f
Branch: refs/heads/master
Commit: 083abd4ff89994682aab406dee18df891a136518
Parents: fab2025
Author: Otto van der Schaaf <os...@we-amp.com>
Authored: Wed Jul 9 22:01:10 2014 +0200
Committer: Otto van der Schaaf <os...@we-amp.com>
Committed: Wed Jul 9 22:27:00 2014 +0200
----------------------------------------------------------------------
plugins/experimental/ats_speed/.gitignore | 15 +
plugins/experimental/ats_speed/LICENSE | 202 ++++
plugins/experimental/ats_speed/Makefile | 66 ++
.../experimental/ats_speed/Makefile.psol_source | 33 +
plugins/experimental/ats_speed/README.md | 52 +
.../experimental/ats_speed/ats_base_fetch.cc | 135 +++
plugins/experimental/ats_speed/ats_base_fetch.h | 80 ++
.../ats_speed/ats_beacon_intercept.cc | 350 ++++++
.../ats_speed/ats_beacon_intercept.h | 23 +
plugins/experimental/ats_speed/ats_config.cc | 196 ++++
plugins/experimental/ats_speed/ats_config.h | 82 ++
.../experimental/ats_speed/ats_demo_filter.cc | 82 ++
.../experimental/ats_speed/ats_demo_filter.h | 57 +
.../experimental/ats_speed/ats_header_utils.cc | 88 ++
.../experimental/ats_speed/ats_header_utils.h | 33 +
.../ats_speed/ats_log_message_handler.cc | 93 ++
.../ats_speed/ats_log_message_handler.h | 28 +
.../ats_speed/ats_message_handler.cc | 107 ++
.../ats_speed/ats_message_handler.h | 68 ++
.../ats_speed/ats_process_context.cc | 76 ++
.../ats_speed/ats_process_context.h | 49 +
.../ats_speed/ats_resource_intercept.cc | 376 ++++++
.../ats_speed/ats_resource_intercept.h | 21 +
.../ats_speed/ats_rewrite_driver_factory.cc | 178 +++
.../ats_speed/ats_rewrite_driver_factory.h | 102 ++
.../ats_speed/ats_rewrite_options.cc | 256 +++++
.../ats_speed/ats_rewrite_options.h | 95 ++
.../ats_speed/ats_server_context.cc | 38 +
.../experimental/ats_speed/ats_server_context.h | 48 +
plugins/experimental/ats_speed/ats_speed.cc | 1083 ++++++++++++++++++
plugins/experimental/ats_speed/ats_speed.h | 94 ++
.../experimental/ats_speed/ats_thread_system.h | 43 +
plugins/experimental/ats_speed/ethread.patch | 13 +
plugins/experimental/ats_speed/gzip/Makefile | 9 +
plugins/experimental/ats_speed/gzip/README | 4 +
.../ats_speed/gzip/configuration.cc | 264 +++++
.../experimental/ats_speed/gzip/configuration.h | 84 ++
.../experimental/ats_speed/gzip/debug_macros.h | 59 +
plugins/experimental/ats_speed/gzip/gzip.cc | 826 +++++++++++++
plugins/experimental/ats_speed/gzip/gzip.config | 6 +
plugins/experimental/ats_speed/gzip/misc.cc | 197 ++++
plugins/experimental/ats_speed/gzip/misc.h | 84 ++
.../ats_speed/scripts/prepare_psol.sh | 95 ++
43 files changed, 5890 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/.gitignore
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/.gitignore b/plugins/experimental/ats_speed/.gitignore
new file mode 100644
index 0000000..a12b1d2
--- /dev/null
+++ b/plugins/experimental/ats_speed/.gitignore
@@ -0,0 +1,15 @@
+# Compiled Object files
+*.slo
+*.lo
+*.o
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.gz
+*~
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/LICENSE
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/LICENSE b/plugins/experimental/ats_speed/LICENSE
new file mode 100644
index 0000000..e06d208
--- /dev/null
+++ b/plugins/experimental/ats_speed/LICENSE
@@ -0,0 +1,202 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/Makefile
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/Makefile b/plugins/experimental/ats_speed/Makefile
new file mode 100644
index 0000000..a2f5de7
--- /dev/null
+++ b/plugins/experimental/ats_speed/Makefile
@@ -0,0 +1,66 @@
+SHELL := /bin/bash
+TSXS?=tsxs
+# Specify BUILDTYPE=Debug for a debug build
+BUILDTYPE?=Release
+MOD_PAGESPEED_DIR=$(shell pwd)/psol/include/
+PAGESPEED_OUT=$(shell pwd)/psol/lib/$(BUILDTYPE)/linux/x64/
+
+
+os_name=unknown_os
+arch_name=ia32
+uname_os=$(shell uname)
+uname_arch=$(shell uname -m)
+
+ifeq ($(uname_os),Linux)
+ os_name=linux
+endif
+
+ifeq ($(uname_arch), x86_64)
+ arch_name=x64
+endif
+ifeq ($(uname_arch), amd64)
+ arch_name=x64
+endif
+
+INC =-I$(MOD_PAGESPEED_DIR)\
+ -I$(MOD_PAGESPEED_DIR)third_party/chromium/src/\
+ -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/src\
+ -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/gen/arch/$(os_name)/$(arch_name)/include\
+ -I$(MOD_PAGESPEED_DIR)third_party/protobuf/src\
+ -I$(MOD_PAGESPEED_DIR)third_party/re2/src\
+ -I$(MOD_PAGESPEED_DIR)third_party/out/$(BUILDTYPE)/obj/gen\
+ -I$(MOD_PAGESPEED_DIR)third_party/apr/src/include/\
+ -I$(MOD_PAGESPEED_DIR)third_party/aprutil/src/include/\
+ -I$(MOD_PAGESPEED_DIR)third_party/apr/gen/arch/$(os_name)/$(arch_name)/include/\
+ -I$(MOD_PAGESPEED_DIR)third_party/aprutil/gen/arch/$(os_name)/$(arch_name)/include/\
+ -I$(MOD_PAGESPEED_DIR)out/$(BUILDTYPE)/obj/gen\
+ -I$(MOD_PAGESPEED_DIR)out/$(BUILDTYPE)/obj/gen/protoc_out/instaweb
+
+PSOL_LIBS = $(PAGESPEED_OUT)pagespeed_automatic.a $(PAGESPEED_OUT)libserf.a $(PAGESPEED_OUT)libaprutil.a $(PAGESPEED_OUT)libapr.a
+
+%.so: psol %.cc
+# https://github.com/pagespeed/ngx_pagespeed/issues/433: it would be nice to have -Wall -Werror, only suppressing when needed.
+ g++ $(INC) -shared -o ats_speed.so -g -pipe -O3 -fpic $(MOD_PAGESPEED_DIR)/out/$(BUILDTYPE)/obj/gen/data2c_out/instaweb/net/instaweb/apache/install/mod_pagespeed_example/*.cc $(MOD_PAGESPEED_DIR)/net/instaweb/system/*.cc *.cc -lstdc++ -lstdc++ -lpthread $(PSOL_LIBS) -lrt
+
+all: psol gzip/gzip.so ats_speed.so
+
+1.7.30.4.tar.gz:
+ wget --no-check-certificate https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
+
+psol/: 1.7.30.4.tar.gz
+ tar -xzvf 1.7.30.4.tar.gz
+
+gzip/gzip.so:
+ cd gzip && make
+
+install: all
+ $(TSXS) -i -o ats_speed.so
+ cd gzip && make install
+
+cleanpsol:
+ rm -rf psol/
+ rm *.gz
+
+clean:
+ rm -f *.lo *.so *.o
+ rm -f gzip/*.lo gzip/*.so gzip/*.o
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/Makefile.psol_source
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/Makefile.psol_source b/plugins/experimental/ats_speed/Makefile.psol_source
new file mode 100755
index 0000000..8428d3a
--- /dev/null
+++ b/plugins/experimental/ats_speed/Makefile.psol_source
@@ -0,0 +1,33 @@
+TSXS?=tsxs
+BUILDTYPE=Release
+MOD_PAGESPEED_DIR=$(HOME)/code/google/mod_pagespeed/src/
+PAGESPEED_OUT=$(MOD_PAGESPEED_DIR)out/$(BUILDTYPE)/
+
+INC =-I$(MOD_PAGESPEED_DIR)\
+ -I$(MOD_PAGESPEED_DIR)third_party/chromium/src/\
+ -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/src\
+ -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/gen/arch/linux/x64/include\
+ -I$(MOD_PAGESPEED_DIR)third_party/protobuf/src\
+ -I$(MOD_PAGESPEED_DIR)third_party/re2/src\
+ -I$(MOD_PAGESPEED_DIR)third_party/out/$(BUILDTYPE)/obj/gen\
+ -I$(MOD_PAGESPEED_DIR)third_party/apr/src/include/\
+ -I$(MOD_PAGESPEED_DIR)third_party/aprutil/src/include/\
+ -I$(MOD_PAGESPEED_DIR)third_party/apr/gen/arch/linux/x64/include/\
+ -I$(MOD_PAGESPEED_DIR)third_party/aprutil/gen/arch/linux/x64/include/\
+ -I$(PAGESPEED_OUT)obj/gen/\
+ -I$(PAGESPEED_OUT)obj/gen/protoc_out/instaweb/
+
+PSOL_LIBS = $(MOD_PAGESPEED_DIR)net/instaweb/automatic/pagespeed_automatic.a
+
+%.so: %.cc
+ g++ $(INC) -shared -o ats_speed.so -g -pipe -Wall -Werror -O3 -fpic *.cc -lstdc++ -lpthread -lrt $(PSOL_LIBS)
+
+all: gzip/gzip.so ats_speed.so
+
+install: all
+ $(TSXS) -i -o ats_speed.so
+ cp gzip/gzip.so ./
+ $(TSXS) -i -o zip.so
+
+clean:
+ rm -f *.lo *.so *.o
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/README.md
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/README.md b/plugins/experimental/ats_speed/README.md
new file mode 100644
index 0000000..ae5db92
--- /dev/null
+++ b/plugins/experimental/ats_speed/README.md
@@ -0,0 +1,52 @@
+
+
+Apache Traffic Server web content optimization plugin powered by Google PageSpeed
+
+http://www.atsspeed.com/
+
+To build, a simple 'make' should work. Use 'sudo make install' to install.
+Optionally, patching ATS with ethread.patch helps with eliminating latency that
+sometimes gets induced when synchronising ATS's and PSOL's thread pools.
+
+After that, update ATS's plugin.config with:
+```
+ats_speed.so
+gzip.so /usr/local/etc/trafficserver/gzip.config
+````
+gzip.so also is build with ats_speed, as it currently is a slightly
+modified version of the official one from the ATS repository.
+
+There are some hard-coded things in the plugin, these directories should exist:
+- /tmp/ps_log/ to exist
+- /tmp/ats_ps/ to exist
+
+Configuration files go into `/usr/local/etc/trafficserver/psol.`
+That folder is monitored, and changes to files in there are picked
+up immediately. A sample configuration:
+
+```
+# [host]
+[192.168.185.185]
+# Force traffic server to cache all origin responses
+override_expiry
+pagespeed FlushHtml on
+pagespeed RewriteLevel CoreFilters
+pagespeed EnableFilters rewrite_domains,trim_urls
+pagespeed MapRewriteDomain http://192.168.185.185 http://www.foo.com
+pagespeed MapOriginDomain http://192.168.185.185 http://www.foo.com
+pagespeed EnableFilters prioritize_critical_css,move_css_to_head,move_css_above_scripts
+pagespeed EnableFilters fallback_rewrite_css_urls,insert_img_dimensions,lazyload_images,local_storage_cache
+pagespeed EnableFilters prioritize_critical_css,rewrite_css
+pagespeed EnableFilters combine_javascript,combine_css
+```
+
+It also expects this in records.config from ATS to function:
+`CONFIG proxy.config.url_remap.pristine_host_hdr INT 0`
+
+You can view debug output of the plugin using `traffic_server -T ".*speed.*"`
+
+The current state compiles against PSOL 1.7.30.4-beta.
+Please note the this plugin will generate asserts when build against
+the debug version of mps (option->Merge from a different thread).
+
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_base_fetch.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_base_fetch.cc b/plugins/experimental/ats_speed/ats_base_fetch.cc
new file mode 100644
index 0000000..7788d8f
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_base_fetch.cc
@@ -0,0 +1,135 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+#include "ats_base_fetch.h"
+
+#include <ts/ts.h>
+
+#include "ats_server_context.h"
+
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/string_writer.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+
+
+using namespace net_instaweb;
+
+// TODO(oschaaf): rename is_resource_fetch -> write_raw_response_headers
+AtsBaseFetch::AtsBaseFetch(AtsServerContext* server_context,
+ const net_instaweb::RequestContextPtr& request_ctx,
+ TSVIO downstream_vio, TSIOBuffer downstream_buffer, bool is_resource_fetch) :
+ AsyncFetch(request_ctx),
+ server_context_(server_context),
+ done_called_(false),
+ last_buf_sent_(false),
+ references_(2),
+ downstream_vio_(downstream_vio),
+ downstream_buffer_(downstream_buffer),
+ is_resource_fetch_(is_resource_fetch),
+ downstream_length_(0),
+ txn_mutex_(TSVIOMutexGet(downstream_vio)) {
+ buffer_.reserve(1024 * 32);
+}
+
+AtsBaseFetch::~AtsBaseFetch() {
+ CHECK(references_ == 0);
+}
+
+// Should be called from the event loop,
+// and thus with the txn mutex held by ATS
+void AtsBaseFetch::Release() {
+ DecrefAndDeleteIfUnreferenced();
+}
+
+void AtsBaseFetch::Lock(){
+ TSMutexLock(txn_mutex_);
+}
+
+void AtsBaseFetch::Unlock() {
+ TSMutexUnlock(txn_mutex_);
+}
+
+bool AtsBaseFetch::HandleWrite(const StringPiece& sp, net_instaweb::MessageHandler* handler) {
+ ForwardData(sp, false, false);
+ return true;
+}
+
+bool AtsBaseFetch::HandleFlush( net_instaweb::MessageHandler* handler ) {
+ ForwardData("", true, false);
+ return true;
+}
+
+void AtsBaseFetch::HandleHeadersComplete() {
+ // oschaaf: ATS will currently send its response headers
+ // earlier than this will fire. So this has become a no-op.
+ // This implies that we can't support convert_meta_tags
+ TSDebug("ats-speed", "HeadersComplete()!");
+ // For resource fetches, we need to output the headers in raw HTTP format.
+ if (is_resource_fetch_) {
+ GoogleMessageHandler mh;
+ GoogleString s;
+ StringWriter string_writer(&s);
+ response_headers()->Add("Connection", "Close");
+ response_headers()->WriteAsHttp(&string_writer, &mh);
+ ForwardData(StringPiece(s.data(),s.size()), true, false);
+ }
+}
+
+void AtsBaseFetch::ForwardData(const StringPiece& sp, bool reenable, bool last) {
+ TSIOBufferBlock downstream_blkp;
+ char *downstream_buffer;
+ int64_t downstream_length;
+ int64_t to_write = sp.size();
+
+ Lock();
+ if (references_ == 2) {
+ while (to_write > 0) {
+ downstream_blkp = TSIOBufferStart(downstream_buffer_);
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+ int64_t bytes_written = to_write > downstream_length ? downstream_length : to_write;
+ memcpy(downstream_buffer, sp.data() + (sp.size() - to_write), bytes_written);
+ to_write -= bytes_written;
+ downstream_length_ += bytes_written;
+ TSIOBufferProduce(downstream_buffer_, bytes_written);
+ }
+ CHECK(to_write == 0) << "to_write failure";
+ if (last) {
+ TSVIONBytesSet(downstream_vio_, downstream_length_);
+ }
+ if (reenable) {
+ TSVIOReenable(downstream_vio_);
+ }
+ }
+ Unlock();
+}
+
+void AtsBaseFetch::HandleDone(bool success) {
+ CHECK(!done_called_);
+ CHECK(downstream_vio_);
+ TSDebug("ats-speed", "Done()!");
+
+ Lock();
+ done_called_ = true;
+ ForwardData("", true, true);
+ DecrefAndDeleteIfUnreferenced();
+ Unlock();
+}
+
+void AtsBaseFetch::DecrefAndDeleteIfUnreferenced() {
+ if (__sync_add_and_fetch(&references_, -1) == 0) {
+ delete this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_base_fetch.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_base_fetch.h b/plugins/experimental/ats_speed/ats_base_fetch.h
new file mode 100644
index 0000000..4d5f88a
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_base_fetch.h
@@ -0,0 +1,80 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_BASE_FETCH_H_
+#define ATS_BASE_FETCH_H_
+
+#include <string>
+
+#include <ts/ts.h>
+
+#include "ats_speed.h"
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/http/public/headers.h"
+#include "net/instaweb/util/public/string.h"
+
+
+namespace net_instaweb {
+
+class AtsServerContext;
+class AbstractMutex;
+
+class AtsBaseFetch : public net_instaweb::AsyncFetch {
+
+public:
+ // TODO(oschaaf): change this to take the downstream buffer and vio
+ // instead of AtsData*. Also, make the bytes send a property
+ // of the fetch itself instead of tracking it on data.
+ // Doing so, would allow us to share this with the server intercept
+ // code for resources.
+ AtsBaseFetch(AtsServerContext* server_context,
+ const net_instaweb::RequestContextPtr& request_ctx,
+ TSVIO downstream_vio,
+ TSIOBuffer downstream_buffer,
+ bool is_resource_fetch);
+
+ virtual ~AtsBaseFetch();
+ void Release();
+private:
+ virtual bool HandleWrite(const StringPiece& sp, net_instaweb::MessageHandler* handler);
+ virtual bool HandleFlush( net_instaweb::MessageHandler* handler);
+ virtual void HandleHeadersComplete();
+ virtual void HandleDone(bool success);
+ void Lock();
+ void Unlock();
+ void DecrefAndDeleteIfUnreferenced();
+ void ForwardData(const StringPiece& sp, bool reenable, bool last);
+ GoogleString buffer_;
+ AtsServerContext* server_context_;
+ bool done_called_;
+ bool last_buf_sent_;
+
+ // How many active references there are to this fetch. Starts at two,
+ // decremented once when Done() is called and once when Release() is called.
+ int references_;
+ TSVIO downstream_vio_;
+ TSIOBuffer downstream_buffer_;
+ bool is_resource_fetch_;
+ int64_t downstream_length_;
+
+ // We don't own this mutex
+ TSMutex txn_mutex_;
+};
+
+} /* ats_pagespeed */
+
+
+#endif /* ATS_BASE_FETCH_H_ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_beacon_intercept.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_beacon_intercept.cc b/plugins/experimental/ats_speed/ats_beacon_intercept.cc
new file mode 100644
index 0000000..175e218
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_beacon_intercept.cc
@@ -0,0 +1,350 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+// 'inspired' by the esi intercept code
+
+#include "ats_beacon_intercept.h"
+#include "ats_speed.h"
+#include "ats_server_context.h"
+
+#include "net/instaweb/system/public/system_request_context.h"
+
+#include <string>
+#include <limits.h>
+#include <strings.h>
+#include <stdio.h>
+
+using std::string;
+using namespace net_instaweb;
+
+#define DEBUG_TAG "ats_speed_beacon"
+
+struct InterceptCtx {
+ TSVConn net_vc;
+ TSCont contp;
+
+ struct IoHandle {
+ TSVIO vio;
+ TSIOBuffer buffer;
+ TSIOBufferReader reader;
+ IoHandle()
+ : vio(0), buffer(0), reader(0) { };
+ ~IoHandle() {
+ if (reader) {
+ TSIOBufferReaderFree(reader);
+ }
+ if (buffer) {
+ TSIOBufferDestroy(buffer);
+ }
+ };
+ };
+
+ IoHandle input;
+ IoHandle output;
+
+ TSHttpParser http_parser;
+ string body;
+ int req_content_len;
+ TSMBuffer req_hdr_bufp;
+ TSMLoc req_hdr_loc;
+ bool req_hdr_parsed;
+ bool initialized;
+ TransformCtx* request_context;
+ InterceptCtx(TSCont cont)
+ : net_vc(0), contp(cont), input(), output(), body(""), req_content_len(0), req_hdr_bufp(0), req_hdr_loc(0),
+ req_hdr_parsed(false), initialized(false) {
+ http_parser = TSHttpParserCreate();
+ }
+
+ bool init(TSVConn vconn);
+
+ void setupWrite();
+
+ ~InterceptCtx() {
+ TSDebug(DEBUG_TAG, "[%s] Destroying continuation data", __FUNCTION__);
+ TSHttpParserDestroy(http_parser);
+ if (req_hdr_loc) {
+ TSHandleMLocRelease(req_hdr_bufp, TS_NULL_MLOC, req_hdr_loc);
+ }
+ if (req_hdr_bufp) {
+ TSMBufferDestroy(req_hdr_bufp);
+ }
+ if (request_context) {
+ ats_ctx_destroy(request_context);
+ request_context = NULL;
+ }
+ };
+};
+
+bool
+InterceptCtx::init(TSVConn vconn)
+{
+ if (initialized) {
+ TSError("[%s] InterceptCtx already initialized!", __FUNCTION__);
+ return false;
+ }
+
+ net_vc = vconn;
+
+ input.buffer = TSIOBufferCreate();
+ input.reader = TSIOBufferReaderAlloc(input.buffer);
+ input.vio = TSVConnRead(net_vc, contp, input.buffer, INT_MAX);
+
+ req_hdr_bufp = TSMBufferCreate();
+ req_hdr_loc = TSHttpHdrCreate(req_hdr_bufp);
+ TSHttpHdrTypeSet(req_hdr_bufp, req_hdr_loc, TS_HTTP_TYPE_REQUEST);
+
+ initialized = true;
+ TSDebug(DEBUG_TAG, "[%s] InterceptCtx initialized!", __FUNCTION__);
+ return true;
+}
+
+void
+InterceptCtx::setupWrite() {
+ TSAssert(output.buffer == 0);
+ output.buffer = TSIOBufferCreate();
+ output.reader = TSIOBufferReaderAlloc(output.buffer);
+ output.vio = TSVConnWrite(net_vc, contp, output.reader, INT_MAX);
+}
+
+// Parses out query params from the request.
+void ps_query_params_handler(StringPiece unparsed_uri, StringPiece* data) {
+ stringpiece_ssize_type question_mark_index = unparsed_uri.find("?");
+ if (question_mark_index == StringPiece::npos) {
+ *data = "";
+ } else {
+ *data = unparsed_uri.substr(
+ question_mark_index+1, unparsed_uri.size() - (question_mark_index+1));
+ }
+}
+
+static bool
+handleRead(InterceptCtx *cont_data, bool &read_complete) {
+ int avail = TSIOBufferReaderAvail(cont_data->input.reader);
+ if (avail == TS_ERROR) {
+ TSError("[%s] Error while getting number of bytes available", __FUNCTION__);
+ return false;
+ }
+
+ TSDebug(DEBUG_TAG, "[%s] Parsed header, avail: %d", __FUNCTION__, avail);
+
+ int consumed = 0;
+ if (avail > 0) {
+ int64_t data_len;
+ const char *data;
+ TSIOBufferBlock block = TSIOBufferReaderStart(cont_data->input.reader);
+ while (block != NULL) {
+ data = TSIOBufferBlockReadStart(block, cont_data->input.reader, &data_len);
+ if (!cont_data->req_hdr_parsed) {
+ const char *endptr = data + data_len;
+ if (TSHttpHdrParseReq(cont_data->http_parser, cont_data->req_hdr_bufp, cont_data->req_hdr_loc,
+ &data, endptr) == TS_PARSE_DONE) {
+ TSDebug(DEBUG_TAG, "[%s] Parsed header", __FUNCTION__);
+ TSMLoc content_len_loc = TSMimeHdrFieldFind(cont_data->req_hdr_bufp, cont_data->req_hdr_loc,
+ TS_MIME_FIELD_CONTENT_LENGTH, -1);
+
+ /*if (!content_len_loc) {
+ TSError("[%s] Error while searching content length header [%s]",
+ __FUNCTION__, TS_MIME_FIELD_CONTENT_LENGTH);
+ return false;
+ }
+ if (!content_len_loc) {
+ TSError("[%s] request doesn't contain content length header [%s]",
+ __FUNCTION__, TS_MIME_FIELD_CONTENT_TYPE);
+ return false;
+ }*/
+ if (!content_len_loc) {
+ cont_data->req_content_len = 0;
+ } else {
+ cont_data->req_content_len = TSMimeHdrFieldValueIntGet(cont_data->req_hdr_bufp, cont_data->req_hdr_loc,
+ content_len_loc, 0);
+ TSHandleMLocRelease(cont_data->req_hdr_bufp, cont_data->req_hdr_loc, content_len_loc);
+ }
+ TSDebug(DEBUG_TAG, "[%s] Got content length as %d", __FUNCTION__, cont_data->req_content_len);
+ if (cont_data->req_content_len < 0) {
+ TSError("[%s] Invalid content length [%d]", __FUNCTION__, cont_data->req_content_len);
+ return false;
+ }
+ if (endptr - data) {
+ TSDebug(DEBUG_TAG, "[%s] Appending %ld bytes to body", __FUNCTION__, static_cast<long int>(endptr - data));
+ cont_data->body.append(data, endptr - data);
+ }
+ cont_data->req_hdr_parsed = true;
+ }
+ } else {
+ //TSDebug(DEBUG_TAG, "[%s] Appending %" PRId64" bytes to body", __FUNCTION__, data_len);
+ cont_data->body.append(data, data_len);
+ }
+ consumed += data_len;
+ block = TSIOBufferBlockNext(block);
+ }
+ }
+
+ TSIOBufferReaderConsume(cont_data->input.reader, consumed);
+
+ TSDebug(DEBUG_TAG, "[%s] Consumed %d bytes from input vio, avail: %d", __FUNCTION__, consumed, avail);
+
+ // Modify the input VIO to reflect how much data we've completed.
+ TSVIONDoneSet(cont_data->input.vio, TSVIONDoneGet(cont_data->input.vio) + consumed);
+
+ if (static_cast<int>(cont_data->body.size()) == cont_data->req_content_len) {
+ TSDebug(DEBUG_TAG, "[%s] Completely read body of size %d", __FUNCTION__, cont_data->req_content_len);
+ read_complete = true;
+ } else {
+ read_complete = false;
+ TSDebug(DEBUG_TAG, "[%s] Reenabling input vio as %ld bytes still need to be read",
+ __FUNCTION__, static_cast<long int>(cont_data->req_content_len - cont_data->body.size()));
+ TSVIOReenable(cont_data->input.vio);
+ }
+ return true;
+}
+
+static bool
+processRequest(InterceptCtx *cont_data) {
+ string reply_header("HTTP/1.1 204 No Content\r\n");
+ int body_size = static_cast<int>(cont_data->body.size());
+ if (cont_data->req_content_len != body_size) {
+ TSError("[%s] Read only %d bytes of body; expecting %d bytes", __FUNCTION__, body_size,
+ cont_data->req_content_len);
+ }
+
+ char buf[64];
+ //snprintf(buf, 64, "%s: %d\r\n\r\n", TS_MIME_FIELD_CONTENT_LENGTH, body_size);
+ snprintf(buf, 64, "%s: %d\r\n\r\n", TS_MIME_FIELD_CONTENT_LENGTH, 0);
+ reply_header.append(buf);
+ reply_header.append("Cache-Control: max-age=0, no-cache");
+ //TSError("[%s] reply header: \n%s", __FUNCTION__, reply_header.data());
+
+ StringPiece query_param_beacon_data;
+ ps_query_params_handler(cont_data->request_context->url_string->c_str(), &query_param_beacon_data);
+
+ GoogleString beacon_data = net_instaweb::StrCat(
+ query_param_beacon_data, "&", cont_data->body);
+ ServerContext* server_context = cont_data->request_context->server_context;
+
+ SystemRequestContext* system_request_context =
+ new SystemRequestContext(server_context->thread_system()->NewMutex(),
+ server_context->timer(),
+ 80,
+ "127.0.0.1");
+
+ if (!server_context->HandleBeacon(
+ beacon_data,
+ cont_data->request_context->user_agent->c_str(),
+ net_instaweb::RequestContextPtr(system_request_context))) {
+ TSError("Beacon handling failure!");
+ } else {
+ //TSDebug(DEBUG_TAG, "Beacon post data processed OK: [%s]", beacon_data.c_str());
+ }
+
+ cont_data->setupWrite();
+ if (TSIOBufferWrite(cont_data->output.buffer, reply_header.data(), reply_header.size()) == TS_ERROR) {
+ TSError("[%s] Error while writing reply header", __FUNCTION__);
+ return false;
+ }
+ /*
+ if (TSIOBufferWrite(cont_data->output.buffer, cont_data->body.data(), body_size) == TS_ERROR) {
+ TSError("[%s] Error while writing content", __FUNCTION__);
+ return false;
+ }*/
+ int total_bytes_written = reply_header.size() + body_size;
+ TSDebug(DEBUG_TAG, "[%s] Wrote reply of size %d", __FUNCTION__, total_bytes_written);
+ TSVIONBytesSet(cont_data->output.vio, total_bytes_written);
+
+ TSVIOReenable(cont_data->output.vio);
+ return true;
+}
+
+static int
+txn_intercept(TSCont contp, TSEvent event, void *edata) {
+ TSDebug(DEBUG_TAG, "[%s] Received event: %d", __FUNCTION__, (int)event);
+
+ InterceptCtx *cont_data = static_cast<InterceptCtx *>(TSContDataGet(contp));
+ bool read_complete = false;
+ bool shutdown = false;
+ switch (event) {
+ case TS_EVENT_NET_ACCEPT:
+ TSDebug(DEBUG_TAG, "[%s] Received net accept event", __FUNCTION__);
+ TSAssert(cont_data->initialized == false);
+ if (!cont_data->init(static_cast<TSVConn>(edata))) {
+ TSError("[%s] Could not initialize continuation data!", __FUNCTION__);
+ return 1;
+ }
+ break;
+ case TS_EVENT_VCONN_READ_READY:
+ TSDebug(DEBUG_TAG, "[%s] Received read ready event", __FUNCTION__);
+ if (!handleRead(cont_data, read_complete)) {
+ TSError("[%s] Error while reading from input vio", __FUNCTION__);
+ //return 0;
+ read_complete = true;
+ }
+ break;
+ case TS_EVENT_VCONN_READ_COMPLETE:
+ case TS_EVENT_VCONN_EOS:
+ // intentional fall-through
+ TSDebug(DEBUG_TAG, "[%s] Received read complete/eos event %d", __FUNCTION__, event);
+ read_complete = true;
+ break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ TSDebug(DEBUG_TAG, "[%s] Received write ready event", __FUNCTION__);
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ TSDebug(DEBUG_TAG, "[%s] Received write complete event", __FUNCTION__);
+ shutdown = true;
+ break;
+ case TS_EVENT_ERROR:
+ // todo: do some error handling here
+ TSDebug(DEBUG_TAG, "[%s] Received error event; going to shutdown, event: %d", __FUNCTION__, event);
+ TSError("[%s] Received error event; going to shutdown, event: %d", __FUNCTION__, event);
+ shutdown = true;
+ break;
+ default:
+ break;
+ }
+
+ if (read_complete) {
+ if (!processRequest(cont_data)) {
+ TSError("[%s] Failed to process process", __FUNCTION__);
+ } else {
+ TSDebug(DEBUG_TAG, "[%s] Processed request successfully", __FUNCTION__);
+ }
+ }
+
+ if (shutdown) {
+ TSDebug(DEBUG_TAG, "[%s] Completed request processing. Shutting down...", __FUNCTION__);
+ if (cont_data->net_vc) {
+ TSVConnClose(cont_data->net_vc);
+ }
+ delete cont_data;
+ TSContDestroy(contp);
+ }
+
+ return 1;
+}
+
+bool
+hook_beacon_intercept(TSHttpTxn txnp) {
+ TSCont contp = TSContCreate(txn_intercept, TSMutexCreate());
+ if (!contp) {
+ TSError("[%s] Could not create intercept request", __FUNCTION__);
+ return false;
+ }
+ InterceptCtx *cont_data = new InterceptCtx(contp);
+ cont_data->request_context = get_transaction_context(txnp);
+ TSContDataSet(contp, cont_data);
+ TSHttpTxnIntercept(contp, txnp);
+ TSDebug(DEBUG_TAG, "[%s] Setup server intercept successfully", __FUNCTION__);
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_beacon_intercept.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_beacon_intercept.h b/plugins/experimental/ats_speed/ats_beacon_intercept.h
new file mode 100644
index 0000000..21a7ea0
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_beacon_intercept.h
@@ -0,0 +1,23 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef _ATS_BEACON_INTERCEPT_H
+#define _ATS_BEACON_INTERCEPT_H
+
+#include "ts/ts.h"
+
+bool hook_beacon_intercept(TSHttpTxn txnp);
+
+#endif
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_config.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_config.cc b/plugins/experimental/ats_speed/ats_config.cc
new file mode 100644
index 0000000..c9dd73d
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_config.cc
@@ -0,0 +1,196 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_config.h"
+
+#include <ts/ts.h>
+#include <fstream>
+
+#include "net/instaweb/util/public/string_util.h"
+
+#include "ats_message_handler.h"
+#include "ats_rewrite_options.h"
+
+namespace net_instaweb {
+
+using namespace std;
+
+
+
+void ltrim_if(string& s, int (* fp) (int)) {
+ for (size_t i = 0; i < s.size();) {
+ if (fp(s[i])) {
+ s.erase(i,1);
+ } else {
+ break;
+ }
+ }
+}
+
+void rtrim_if(string& s, int (* fp) (int)) {
+ for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) {
+ if (fp(s[i])) {
+ s.erase(i,1);
+ } else {
+ break;
+ }
+ }
+}
+
+void trim_if(string& s, int (* fp) (int)) {
+ ltrim_if(s, fp);
+ rtrim_if(s, fp);
+}
+
+vector<string> tokenize(const string &s, int (* fp) (int)) {
+ vector<string> r;
+ string tmp;
+
+ for (size_t i = 0; i < s.size(); i++) {
+ if ( fp(s[i]) ) {
+ if ( tmp.size() ) {
+ r.push_back(tmp);
+ tmp = "";
+ }
+ } else {
+ tmp += s[i];
+ }
+ }
+
+ if ( tmp.size() ) {
+ r.push_back(tmp);
+ }
+
+ return r;
+}
+
+AtsConfig::AtsConfig(AtsThreadSystem* thread_system)
+ : thread_system_(thread_system) {
+ AddHostConfig(new AtsHostConfig(GoogleString("(XXXXXX)"), new AtsRewriteOptions(thread_system_)));
+}
+
+AtsConfig::~AtsConfig() {
+ for (size_t i = 0; i < host_configurations_.size(); i++) {
+ delete host_configurations_[i];
+ host_configurations_.clear();
+ }
+}
+
+void AtsConfig::AddHostConfig(AtsHostConfig* hc){
+ host_configurations_.push_back(hc);
+}
+
+AtsHostConfig::~AtsHostConfig() {
+ if (options_ != NULL) {
+ delete options_;
+ options_ = NULL;
+ }
+}
+
+AtsHostConfig * AtsConfig::Find(const char * host, int host_length) {
+ AtsHostConfig * host_configuration = host_configurations_[0];
+
+ std::string shost(host, host_length);
+
+ for (size_t i = 1; i < host_configurations_.size(); i++ ) {
+ if (host_configurations_[i]->host() == shost){
+ host_configuration = host_configurations_[i];
+ break;
+ }
+ }
+
+ return host_configuration;
+}
+
+bool AtsConfig::Parse(const char * path ) {
+ string pathstring(path);
+
+ // If we have a path and it's not an absolute path, make it relative to the
+ // configuration directory.
+ if (!pathstring.empty() && pathstring[0] != '/') {
+ pathstring.assign(TSConfigDirGet());
+ pathstring.append("/");
+ pathstring.append(path);
+ }
+
+ trim_if(pathstring, isspace);
+
+ AtsHostConfig* current_host_configuration = host_configurations_[0];
+
+ if (pathstring.empty()) {
+ TSError("Empty path passed in AtsConfig::Parse");
+ return false;
+ }
+
+ path = pathstring.c_str();
+ std::ifstream f;
+
+ size_t lineno = 0;
+
+ f.open(path, std::ios::in);
+
+ if (!f.is_open()) {
+ TSError("could not open file [%s], skip",path);
+ return false;
+ }
+
+
+ while (!f.eof()) {
+ std::string line;
+ getline(f, line);
+ ++lineno;
+
+ trim_if(line, isspace);
+ if (line.size() == 0) {
+ continue;
+ }
+ if (line[0] == '#') {
+ continue;
+ }
+
+ vector<string> v = tokenize( line, isspace );
+ if (v.size() == 0)
+ continue;
+ GoogleString msg;
+ AtsMessageHandler handler(thread_system_->NewMutex());
+ if (v.size() == 1) {
+ string token = v[0];
+ if ((token[0] == '[') && (token[token.size()-1] == ']')) {
+ GoogleString current_host = token.substr(1, token.size() - 2);
+ current_host_configuration = new AtsHostConfig(current_host, new AtsRewriteOptions(thread_system_));
+ AddHostConfig(current_host_configuration);
+ } else if (StringCaseEqual(token,"override_expiry")) {
+ current_host_configuration->set_override_expiry(true);
+ } else {
+ msg = "unknown single token on a line";
+ }
+ } else {
+ global_settings settings;
+ v.erase (v.begin());
+ const char* err = current_host_configuration->options()->ParseAndSetOptions(v, &handler, settings);
+ if (err) {
+ msg.append(err);
+ }
+ }
+ if (msg.size() > 0) {
+ TSDebug("ats-speed", "Error parsing line [%s]: [%s]", line.c_str(), msg.c_str());
+ }
+ }
+
+ return true;
+}
+
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_config.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_config.h b/plugins/experimental/ats_speed/ats_config.h
new file mode 100644
index 0000000..bc6e7f8
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_config.h
@@ -0,0 +1,82 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_CONFIG_H_
+#define ATS_CONFIG_H_
+
+#include <string>
+#include <vector>
+
+#include <ts/ts.h>
+
+#include "ats_thread_system.h"
+
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+
+namespace net_instaweb {
+
+class AtsRewriteOptions;
+
+class AtsHostConfig {
+public:
+ explicit AtsHostConfig(const GoogleString & host, AtsRewriteOptions* options)
+ : host_(host)
+ , options_(options)
+ {
+ }
+ virtual ~AtsHostConfig();
+
+ inline GoogleString host() { return host_; }
+ inline AtsRewriteOptions* options() { return options_; }
+ inline bool override_expiry() { return override_expiry_; }
+ inline void set_override_expiry(bool x) { override_expiry_ = x; }
+private:
+ GoogleString host_;
+ AtsRewriteOptions* options_;
+ bool override_expiry_;
+ DISALLOW_COPY_AND_ASSIGN(AtsHostConfig);
+}; // class AtsHostConfig
+
+class AtsConfig {
+ friend class AtsHostConfig;
+public:
+ explicit AtsConfig(AtsThreadSystem* thread_system);
+ virtual ~AtsConfig();
+
+ // TODO(oschaaf): destructor??
+ bool Parse(const char * path);
+ AtsHostConfig * Find(const char * host, int host_length);
+ inline AtsHostConfig * GlobalConfiguration() {
+ return host_configurations_[0];
+ }
+ AtsThreadSystem* thread_system() {
+ return thread_system_;
+ }
+
+private:
+ void AddHostConfig(AtsHostConfig* hc);
+
+ std::vector<AtsHostConfig *> host_configurations_;
+ AtsThreadSystem* thread_system_;
+ //todo: destructor. delete owned host configurations
+ DISALLOW_COPY_AND_ASSIGN(AtsConfig);
+}; // class Configuration
+
+
+} // namespace net_instaweb
+
+#endif // ATS_CONFIG_H
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_demo_filter.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_demo_filter.cc b/plugins/experimental/ats_speed/ats_demo_filter.cc
new file mode 100644
index 0000000..18944d5
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_demo_filter.cc
@@ -0,0 +1,82 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "net/instaweb/rewriter/public/add_head_filter.h"
+#include "net/instaweb/htmlparse/public/html_parse.h"
+#include "net/instaweb/htmlparse/public/html_element.h"
+#include <string>
+
+#include <ts/ts.h>
+#include "ats_demo_filter.h"
+
+namespace net_instaweb {
+ const char* AtsDemoFilter::kPoweredByHtml =
+ "<div id=\"weamp_poweredby\" style=\"bottom:0; height:30px; left:0; width:100%;\">"
+ "<div style=\"line-height:30px; margin:0 auto; width:100%; text-align:center; \">"
+ "<a target=\"_blank\" title=\"Google PageSpeed optimization demo brought to you by We-Amp\" href=\"http://www.we-amp.com/\">Google PageSpeed optimization demo by We-Amp</a>"
+ "</div>"
+ "</div>";
+
+
+
+AtsDemoFilter::AtsDemoFilter(HtmlParse* parser, bool banner) :
+ parser_(parser),
+ banner_(banner)
+{
+}
+
+void AtsDemoFilter::StartElement(HtmlElement* element) {
+ if (banner_ && element->keyword() == HtmlName::kBody) {
+ HtmlNode* el = parser_->NewCharactersNode(NULL, AtsDemoFilter::kPoweredByHtml);
+ parser_->InsertNodeBeforeCurrent(el);
+ }
+
+ if (element->keyword() == HtmlName::kA || element->keyword() == HtmlName::kBase
+ || element->keyword() == HtmlName::kForm|| element->keyword() == HtmlName::kImg
+ || element->keyword() == HtmlName::kLink || element->keyword() == HtmlName::kScript) {
+ HtmlElement::AttributeList* attributes = element->mutable_attributes();
+ for (HtmlElement::AttributeIterator i(attributes->begin());
+ i != attributes->end(); ++i) {
+
+ HtmlElement::Attribute& attribute = *i;
+ if (attribute.keyword() == HtmlName::kAction || attribute.keyword() == HtmlName::kHref
+ || attribute.keyword() == HtmlName::kSrc) {
+ const char * attribute_value = NULL;
+ if ( attribute.DecodedValueOrNull() != NULL ) {
+ attribute_value = attribute.DecodedValueOrNull();
+ } else {
+ attribute_value = attribute.escaped_value();
+ }
+
+ if ( attribute_value != NULL) {
+ GoogleUrl url( attribute_value );
+ if (url.IsWebValid()) {
+ if (url.Host() == from_domain_) {
+ StringPiece scheme = url.Scheme();
+ StringPiece host = to_domain_.c_str();
+ StringPiece pathAndQuery = url.PathAndLeaf();
+ GoogleString rewritten = StrCat(scheme,"://", host, pathAndQuery);
+ attribute.SetValue(rewritten.c_str());
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_demo_filter.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_demo_filter.h b/plugins/experimental/ats_speed/ats_demo_filter.h
new file mode 100644
index 0000000..dc5aa78
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_demo_filter.h
@@ -0,0 +1,57 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_DEMO_FILTER_H_
+#define ATS_DEMO_FILTER_H_
+
+#include "base/basictypes.h"
+#include "base/string_piece.h"
+#include "net/instaweb/htmlparse/public/empty_html_filter.h"
+#include "net/instaweb/util/public/atom.h"
+#include "net/instaweb/htmlparse/public/html_parse.h"
+#include "net/instaweb/htmlparse/public/html_element.h"
+#include <string>
+
+using base::StringPiece;
+
+namespace net_instaweb {
+
+
+
+ class AtsDemoFilter : public EmptyHtmlFilter {
+ public:
+ static const char* kPoweredByHtml;
+
+ explicit AtsDemoFilter(HtmlParse* parser, bool banner);
+ virtual void StartElement(HtmlElement* element);
+ virtual const char* Name() const { return "AtsDemo"; }
+ // TODO: move to constructor
+ void set_domains(const StringPiece& to_domain, const StringPiece& from_domain)
+ {
+ to_domain.CopyToString(&to_domain_);
+ from_domain.CopyToString(&from_domain_);
+ }
+
+ private:
+ std::string to_domain_;
+ std::string from_domain_;
+ HtmlParse* parser_;
+ bool banner_;
+ DISALLOW_COPY_AND_ASSIGN(AtsDemoFilter);
+ };
+
+} // namespace net_instaweb
+
+#endif // ATS_DEMO_FILTER_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_header_utils.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_header_utils.cc b/plugins/experimental/ats_speed/ats_header_utils.cc
new file mode 100644
index 0000000..74e46c9
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_header_utils.cc
@@ -0,0 +1,88 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_header_utils.h"
+
+GoogleString get_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name)
+{
+ const char * val = NULL;
+ int val_len;
+ TSMLoc field_loc = TSMimeHdrFieldFind( bufp, hdr_loc, header_name, -1);
+
+ if (field_loc) {
+ val = TSMimeHdrFieldValueStringGet (bufp, hdr_loc, field_loc, 0, &val_len);
+ TSHandleMLocRelease(bufp,hdr_loc,field_loc);
+ return GoogleString(val,val_len);
+ }
+
+ return GoogleString("");
+}
+
+void unset_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name)
+{
+ TSMLoc field_loc = TSMimeHdrFieldFind( bufp, hdr_loc, header_name, -1);
+
+ if (field_loc) {
+ TSMimeHdrFieldDestroy(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ }
+}
+
+void hide_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ while (field) {
+ TSMLoc tmp;
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+}
+
+void restore_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
+
+ while (field) {
+ TSMLoc tmp;
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+}
+
+void set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name, const char * header_value)
+{
+ TSMLoc field_loc = TSMimeHdrFieldFind( bufp, hdr_loc, header_name, -1);
+
+ if (field_loc) {
+ TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, header_value, -1);
+ } else {
+ if ( TSMimeHdrFieldCreate(bufp, hdr_loc, &field_loc) == TS_SUCCESS ) {
+ TSMimeHdrFieldNameSet(bufp, hdr_loc, field_loc, header_name, -1);
+ TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc);
+ TSMimeHdrFieldValueStringSet(bufp,hdr_loc,field_loc,-1,header_value,-1);
+ } else {
+ TSError("field creation error for field [%s]", header_name);
+ return;
+ }
+ }
+
+ if (field_loc) {
+ TSHandleMLocRelease(bufp,hdr_loc,field_loc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_header_utils.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_header_utils.h b/plugins/experimental/ats_speed/ats_header_utils.h
new file mode 100644
index 0000000..7aa6f7d
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_header_utils.h
@@ -0,0 +1,33 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_HEADER_UTILS_H
+#define ATS_HEADER_UTILS_H
+
+#include <string>
+
+#include <ts/ts.h>
+
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+
+GoogleString get_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name);
+void unset_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name);
+void hide_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
+void restore_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
+void set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name, const char * header_value);
+
+#endif // ATS_HEADER_UTILS_H
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_log_message_handler.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_log_message_handler.cc b/plugins/experimental/ats_speed/ats_log_message_handler.cc
new file mode 100644
index 0000000..635a841
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_log_message_handler.cc
@@ -0,0 +1,93 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_log_message_handler.h"
+
+#include <ts/ts.h>
+
+#include <unistd.h>
+
+#include <limits>
+#include <string>
+
+#include "base/debug/debugger.h"
+#include "base/debug/stack_trace.h"
+#include "base/logging.h"
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/util/public/string_util.h"
+
+// Make sure we don't attempt to use LOG macros here, since doing so
+// would cause us to go into an infinite log loop.
+#undef LOG
+#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION
+
+namespace {
+
+bool LogMessageHandler(int severity, const char* file, int line,
+ size_t message_start, const GoogleString& str) {
+ GoogleString message = str;
+ if (severity == logging::LOG_FATAL) {
+ if (base::debug::BeingDebugged()) {
+ base::debug::BreakDebugger();
+ } else {
+ base::debug::StackTrace trace;
+ std::ostringstream stream;
+ trace.OutputToStream(&stream);
+ message.append(stream.str());
+ }
+ }
+
+ // Trim the newline off the end of the message string.
+ size_t last_msg_character_index = message.length() - 1;
+ if (message[last_msg_character_index] == '\n') {
+ message.resize(last_msg_character_index);
+ }
+
+ TSDebug("ats-speed-vlog", "[%s] %s",
+ net_instaweb::kModPagespeedVersion,
+ message.c_str());
+
+ if (severity == logging::LOG_FATAL) {
+ // Crash the process to generate a dump.
+ base::debug::BreakDebugger();
+ }
+
+ return true;
+}
+
+} // namespace
+
+
+namespace net_instaweb {
+
+namespace log_message_handler {
+
+
+const int kDebugLogLevel = -2;
+
+void Install() {
+ logging::SetLogMessageHandler(&LogMessageHandler);
+
+ // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
+ // level is DEBUG.
+ // TODO(oschaaf): from config
+ //if (log->log_level >= NGX_LOG_DEBUG) {
+ logging::SetMinLogLevel(-2);
+ //}
+}
+
+} // namespace log_message_handler
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_log_message_handler.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_log_message_handler.h b/plugins/experimental/ats_speed/ats_log_message_handler.h
new file mode 100644
index 0000000..1a113aa
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_log_message_handler.h
@@ -0,0 +1,28 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_LOG_MESSAGE_HANDLER_H_
+#define ATS_LOG_MESSAGE_HANDLER_H_
+
+
+namespace net_instaweb {
+
+ namespace log_message_handler {
+ void Install();
+ } // namespace log_message_handler
+
+} // namespace net_instaweb
+
+#endif // ATS_LOG_MESSAGE_HANDLER_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_message_handler.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_message_handler.cc b/plugins/experimental/ats_speed/ats_message_handler.cc
new file mode 100644
index 0000000..7f21660
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_message_handler.cc
@@ -0,0 +1,107 @@
+// Copyright 2013 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#include "ats_message_handler.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "net/instaweb/util/public/abstract_mutex.h"
+#include "net/instaweb/util/public/debug.h"
+#include "net/instaweb/util/public/shared_circular_buffer.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/time_util.h"
+
+
+namespace {
+
+// This will be prefixed to every logged message.
+const char kModuleName[] = "ats_pagespeed";
+
+} // namespace
+
+namespace net_instaweb {
+
+AtsMessageHandler::AtsMessageHandler(AbstractMutex* mutex)
+ : mutex_(mutex),
+ buffer_(NULL) {
+ SetPidString(static_cast<int64>(getpid()));
+}
+
+
+bool AtsMessageHandler::Dump(Writer* writer) {
+ // Can't dump before SharedCircularBuffer is set up.
+ if (buffer_ == NULL) {
+ return false;
+ }
+ return buffer_->Dump(writer, &handler_);
+}
+
+void AtsMessageHandler::set_buffer(SharedCircularBuffer* buff) {
+ ScopedMutex lock(mutex_.get());
+ buffer_ = buff;
+}
+
+void AtsMessageHandler::MessageVImpl(MessageType type, const char* msg,
+ va_list args) {
+ GoogleString formatted_message = Format(msg, args);
+
+ TSDebug("ats-speed", "[%s %s] %s", kModuleName, kModPagespeedVersion,
+ formatted_message.c_str());
+
+ // Prepare a log message for the SharedCircularBuffer only.
+ // Prepend time and severity to message.
+ // Format is [time] [severity] [pid] message.
+ GoogleString message;
+ GoogleString time;
+ PosixTimer timer;
+ if (!ConvertTimeToString(timer.NowMs(), &time)) {
+ time = "?";
+ }
+
+ StrAppend(&message, "[", time, "] ",
+ "[", MessageTypeToString(type), "] ");
+ StrAppend(&message, pid_string_, " ", formatted_message, "\n");
+ {
+ ScopedMutex lock(mutex_.get());
+ if (buffer_ != NULL) {
+ buffer_->Write(message);
+ }
+ }
+}
+
+void AtsMessageHandler::FileMessageVImpl(MessageType type, const char* file,
+ int line, const char* msg,
+ va_list args) {
+ GoogleString formatted_message = Format(msg, args);
+ TSDebug("ats-speed", "[%s %s] %s:%d:%s",
+ kModuleName, kModPagespeedVersion, file, line,
+ formatted_message.c_str());
+}
+
+// TODO(sligocki): It'd be nice not to do so much string copying.
+GoogleString AtsMessageHandler::Format(const char* msg, va_list args) {
+ GoogleString buffer;
+
+ // Ignore the name of this routine: it formats with vsnprintf.
+ // See base/stringprintf.cc.
+ StringAppendV(&buffer, msg, args);
+ return buffer;
+}
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_message_handler.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_message_handler.h b/plugins/experimental/ats_speed/ats_message_handler.h
new file mode 100644
index 0000000..fb40590
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_message_handler.h
@@ -0,0 +1,68 @@
+// Copyright 2013 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#ifndef NGX_MESSAGE_HANDLER_H_
+#define NGX_MESSAGE_HANDLER_H_
+
+#include <ts/ts.h>
+#include <cstdarg>
+
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+ class AbstractMutex;
+ class SharedCircularBuffer;
+ class Timer;
+ class Writer;
+
+ class AtsMessageHandler : public GoogleMessageHandler {
+ public:
+ explicit AtsMessageHandler(AbstractMutex* mutex);
+
+ void set_buffer(SharedCircularBuffer* buff);
+
+ void SetPidString(const int64 pid) {
+ pid_string_ = StrCat("[", Integer64ToString(pid), "]");
+ }
+ // Dump contents of SharedCircularBuffer.
+ bool Dump(Writer* writer);
+
+ protected:
+ virtual void MessageVImpl(MessageType type, const char* msg, va_list args);
+
+ virtual void FileMessageVImpl(MessageType type, const char* filename,
+ int line, const char* msg, va_list args);
+
+ private:
+ GoogleString Format(const char* msg, va_list args);
+
+ scoped_ptr<AbstractMutex> mutex_;
+ GoogleString pid_string_;
+ GoogleMessageHandler handler_;
+ SharedCircularBuffer* buffer_;
+
+ DISALLOW_COPY_AND_ASSIGN(AtsMessageHandler);
+ };
+
+} // namespace net_instaweb
+
+#endif // NGX_MESSAGE_HANDLER_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_process_context.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_process_context.cc b/plugins/experimental/ats_speed/ats_process_context.cc
new file mode 100644
index 0000000..2cc47a7
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_process_context.cc
@@ -0,0 +1,76 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_process_context.h"
+
+#include <vector>
+
+#include "ats_rewrite_driver_factory.h"
+#include "ats_server_context.h"
+#include "ats_message_handler.h"
+#include "ats_thread_system.h"
+
+#include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/util/public/pthread_shared_mem.h"
+
+namespace net_instaweb {
+
+AtsProcessContext::AtsProcessContext() {
+ AtsThreadSystem* ts = new AtsThreadSystem();
+ message_handler_.reset(new AtsMessageHandler(ts->NewMutex()));
+ driver_factory_.reset(new AtsRewriteDriverFactory(ts));
+ server_context_ = driver_factory()->MakeAtsServerContext();
+
+ AtsRewriteOptions* root_options_ = (AtsRewriteOptions*)driver_factory_->default_options();
+ AtsRewriteOptions* server_options = root_options_->Clone();
+ AtsRewriteOptions* options = new AtsRewriteOptions(driver_factory_->thread_system());
+ server_options->Merge(*options);
+ delete options;
+
+ server_context_->global_options()->Merge(*server_options);
+ delete server_options;
+
+ message_handler_->Message(kInfo,"global default options:\r\n[%s]",driver_factory_->default_options()->OptionsToString().c_str());
+ message_handler_->Message(kInfo,"server ctx default options:\r\n[%s]",server_context_->global_options()->OptionsToString().c_str());
+ std::vector<SystemServerContext*> server_contexts;
+ server_contexts.push_back(server_context_);
+
+ //Statistics* statistics =
+ // driver_factory_->MakeGlobalSharedMemStatistics(*(SystemRewriteOptions*)server_context_->global_options());
+ GoogleString error_message;
+ int error_index = -1;
+ Statistics* global_statistics = NULL;
+ driver_factory_.get()->PostConfig(
+ server_contexts, &error_message, &error_index, &global_statistics);
+ if (error_index != -1) {
+ server_contexts[error_index]->message_handler()->Message(
+ kError, "ngx_pagespeed is enabled. %s", error_message.c_str());
+ //return NGX_ERROR;
+ CHECK(false);
+ }
+
+ AtsRewriteDriverFactory::InitStats(global_statistics);
+
+ driver_factory()->RootInit();
+ driver_factory()->ChildInit();
+
+ proxy_fetch_factory_.reset(new ProxyFetchFactory(server_context_));
+ message_handler_->Message(kInfo, "Process context constructed");
+}
+
+AtsProcessContext::~AtsProcessContext() {
+}
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_process_context.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_process_context.h b/plugins/experimental/ats_speed/ats_process_context.h
new file mode 100644
index 0000000..aae2118
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_process_context.h
@@ -0,0 +1,49 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_PROCESS_CONTEXT_H_
+#define ATS_PROCESS_CONTEXT_H_
+
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
+
+namespace net_instaweb {
+
+class AtsRewriteDriverFactory;
+class ProxyFetchFactory;
+class AtsServerContext;
+
+class AtsProcessContext {
+ public:
+ explicit AtsProcessContext();
+ virtual ~AtsProcessContext();
+
+ // TODO(oschaaf): const correctness
+ MessageHandler* message_handler() { return message_handler_.get(); }
+ AtsRewriteDriverFactory* driver_factory() { return driver_factory_.get(); }
+ ProxyFetchFactory* proxy_fetch_factory() { return proxy_fetch_factory_.get(); }
+ AtsServerContext* server_context() { return server_context_; }
+ private:
+ scoped_ptr<MessageHandler> message_handler_;
+ scoped_ptr<AtsRewriteDriverFactory> driver_factory_;
+ scoped_ptr<ProxyFetchFactory> proxy_fetch_factory_;
+ AtsServerContext* server_context_;
+};
+
+
+} // namespace net_instaweb
+
+#endif // ATS_PROCESS_CONTEXT_H_
[2/3] ats_speed: PageSpeed optimization plugin
Posted by os...@apache.org.
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_resource_intercept.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_resource_intercept.cc b/plugins/experimental/ats_speed/ats_resource_intercept.cc
new file mode 100644
index 0000000..e54d15d
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_resource_intercept.cc
@@ -0,0 +1,376 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include <ts/ts.h>
+
+#include <stdio.h>
+
+#include "ats_resource_intercept.h"
+
+
+#include "ats_base_fetch.h"
+#include "ats_rewrite_driver_factory.h"
+#include "ats_rewrite_options.h"
+#include "ats_server_context.h"
+#include "ats_speed.h"
+
+#include "net/instaweb/http/public/request_context.h"
+#include "net/instaweb/rewriter/public/resource_fetch.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/system_request_context.h"
+
+#include "net/instaweb/util/public/string_writer.h"
+#include "net/instaweb/system/public/handlers.h"
+
+using namespace net_instaweb;
+
+struct InterceptCtx
+{
+ TSVConn vconn;
+ TSIOBuffer req_buffer;
+ TSIOBufferReader req_reader;
+ TSIOBuffer resp_buffer;
+ TSIOBufferReader resp_reader;
+ GoogleString* response;
+ TransformCtx* request_ctx;
+ RequestHeaders* request_headers;
+
+ InterceptCtx()
+ : vconn(NULL)
+ , req_buffer(NULL)
+ , req_reader(NULL)
+ , resp_buffer(NULL)
+ , resp_reader(NULL)
+ , response( new GoogleString() )
+ , request_ctx(NULL)
+ , request_headers(NULL)
+ {
+ };
+};
+
+static void
+shutdown (TSCont cont, InterceptCtx * intercept_ctx) {
+ if (intercept_ctx->req_reader != NULL) {
+ TSIOBufferReaderFree(intercept_ctx->req_reader);
+ intercept_ctx->req_reader = NULL;
+ }
+ if (intercept_ctx->req_buffer != NULL) {
+ TSIOBufferDestroy(intercept_ctx->req_buffer);
+ intercept_ctx->req_buffer = NULL;
+ }
+ if (intercept_ctx->resp_reader != NULL) {
+ TSIOBufferReaderFree(intercept_ctx->resp_reader);
+ intercept_ctx->resp_reader = NULL;
+ }
+ if (intercept_ctx->resp_buffer != NULL) {
+ TSIOBufferDestroy(intercept_ctx->resp_buffer);
+ intercept_ctx->resp_buffer = NULL;
+ }
+ if (intercept_ctx->vconn != NULL) {
+ TSVConnShutdown(intercept_ctx->vconn, 0, 1);
+ TSVConnClose(intercept_ctx->vconn);
+ intercept_ctx->vconn = NULL;
+ }
+ if (intercept_ctx->response != NULL) {
+ delete intercept_ctx->response;
+ intercept_ctx->response = NULL;
+ }
+ // TODO(oschaaf): think the ordering of this one through.
+ if (intercept_ctx->request_ctx) {
+ ats_ctx_destroy(intercept_ctx->request_ctx);
+ intercept_ctx->request_ctx = NULL;
+ }
+ if (intercept_ctx->request_headers != NULL) {
+ delete intercept_ctx->request_headers;
+ intercept_ctx->request_headers = NULL;
+ }
+ delete intercept_ctx;
+ TSContDestroy(cont);
+}
+
+static int
+resource_intercept(TSCont cont, TSEvent event, void *edata)
+{
+ InterceptCtx *intercept_ctx = static_cast<InterceptCtx *>(TSContDataGet(cont));
+ bool shutDown = false;
+
+ // TODO(oschaaf): have a look at https://github.com/apache/trafficserver/blob/master/plugins/experimental/esi/serverIntercept.c
+ // and see if we have any edge cases we should fix.
+ switch (event) {
+ case TS_EVENT_NET_ACCEPT: {
+ intercept_ctx->vconn = static_cast<TSVConn>(edata);
+ intercept_ctx->req_buffer = TSIOBufferCreate();
+ intercept_ctx->req_reader = TSIOBufferReaderAlloc(intercept_ctx->req_buffer);
+ intercept_ctx->resp_buffer = TSIOBufferCreate();
+ intercept_ctx->resp_reader = TSIOBufferReaderAlloc(intercept_ctx->resp_buffer);
+ TSVConnRead(intercept_ctx->vconn, cont, intercept_ctx->req_buffer, 0x7fffffff);
+ } break;
+ case TS_EVENT_VCONN_READ_READY: {
+ CHECK(intercept_ctx->request_ctx->base_fetch == NULL) << "Base fetch must not be set!";
+ CHECK(intercept_ctx->request_ctx->url_string != NULL) << "Url must be set!";
+
+ TSVConnShutdown(intercept_ctx->vconn, 1, 0);
+
+ // response will already have a size for internal pages at this point.
+ // resources, however, will have to be fetched.
+ // TODO(oschaaf): this is extremely ugly.
+ if (intercept_ctx->response->size() == 0) {
+ // TODO(oschaaf): unused - must we close / clean this up?
+ TSVIO downstream_vio = TSVConnWrite(
+ intercept_ctx->vconn, cont, intercept_ctx->resp_reader, 0x7fffffff);
+
+ AtsServerContext* server_context = intercept_ctx->request_ctx->server_context;
+
+ // TODO:(oschaaf) host/port
+ SystemRequestContext* system_request_context =
+ new SystemRequestContext(server_context->thread_system()->NewMutex(),
+ server_context->timer(),
+ 80,
+ "127.0.0.1");
+
+ intercept_ctx->request_ctx->base_fetch = new AtsBaseFetch(
+ server_context, RequestContextPtr(system_request_context),
+ downstream_vio, intercept_ctx->resp_buffer, true);
+ intercept_ctx->request_ctx->base_fetch->set_request_headers(
+ intercept_ctx->request_headers);
+
+ RewriteOptions* options = NULL;
+
+ //const char* host = intercept_ctx->request_headers->Lookup1(HttpAttributes::kHost);
+ const char* host = intercept_ctx->request_ctx->gurl->HostAndPort().as_string().c_str();
+ if (host != NULL && strlen(host) > 0) {
+ intercept_ctx->request_ctx->options = get_host_options(host);
+ }
+
+ // TODO(oschaaf): directory options should be coming from configuration!
+ bool ok = ps_determine_options(server_context,
+ intercept_ctx->request_ctx->options,
+ intercept_ctx->request_ctx->base_fetch->request_headers(),
+ intercept_ctx->request_ctx->base_fetch->response_headers(),
+ &options,
+ intercept_ctx->request_ctx->gurl);
+
+ // Take ownership of custom_options.
+ scoped_ptr<RewriteOptions> custom_options(options);
+
+ if (!ok) {
+ TSError("Failure while determining request options for psol resource");
+ // options = server_context->global_options();
+ } else {
+ // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
+ // parameters. Keep url_string in sync with url.
+ // TODO(oschaaf): we really should determine if we have to do the lookup
+ intercept_ctx->request_ctx->gurl->Spec().CopyToString(intercept_ctx->request_ctx->url_string);
+ }
+
+ // The url we have here is already checked for IsWebValid()
+ net_instaweb::ResourceFetch::Start(
+ GoogleUrl(*intercept_ctx->request_ctx->url_string),
+ custom_options.release() /* null if there aren't custom options */,
+ false /* using_spdy */, server_context, intercept_ctx->request_ctx->base_fetch);
+ } else {
+ int64_t numBytesToWrite, numBytesWritten;
+ numBytesToWrite = intercept_ctx->response->size();
+ numBytesWritten = TSIOBufferWrite(intercept_ctx->resp_buffer,
+ intercept_ctx->response->c_str(), numBytesToWrite);
+
+ if (numBytesWritten == numBytesToWrite) {
+ TSVConnWrite(intercept_ctx->vconn, cont, intercept_ctx->resp_reader, numBytesToWrite);
+ } else {
+ TSError("Not all output could be written in one go");
+ DCHECK(false);
+ }
+ }
+ } break;
+ case TS_EVENT_VCONN_EOS:
+ TSVConnShutdown(intercept_ctx->vconn, 1, 0);
+ break;
+ case TS_EVENT_VCONN_READ_COMPLETE: {
+ TSVConnShutdown(intercept_ctx->vconn, 1, 0);
+ } break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ shutDown = true;
+ break;
+ case TS_EVENT_ERROR:
+ TSError("vconn event: error %s", intercept_ctx->request_ctx->url_string->c_str());
+ shutDown = true;
+ break;
+ case TS_EVENT_NET_ACCEPT_FAILED:
+ TSError("vconn event: accept failed");
+ shutDown = true;
+ break;
+ case TS_EVENT_IMMEDIATE:
+ case TS_EVENT_TIMEOUT:
+ break;
+ default:
+ TSError("default clause event: %d", event);
+ break;
+ }
+
+ if (shutDown) {
+ shutdown(cont, intercept_ctx);
+ }
+
+ return 1;
+}
+
+// We intercept here because serving from ats's own cache is faster
+// then serving from pagespeed's cache. (which needs to be looked in to)
+static int
+read_cache_header_callback(TSCont cont, TSEvent event, void *edata)
+{
+ TSHttpTxn txn = static_cast<TSHttpTxn>(edata);
+ TransformCtx* ctx = get_transaction_context(txn);
+
+ if (ctx == NULL) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ if (!ctx->resource_request) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ // TODO(oschaaf): FIXME: Ownership of ctx has become too mucky.
+ // This is because I realised too late that the intercepts
+ // are able to outlive the transaction, which I hacked
+ // to work.
+ if (TSHttpIsInternalRequest(txn) == TS_SUCCESS) {
+ ats_ctx_destroy(ctx);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ if (cache_hit(txn)) {
+ ats_ctx_destroy(ctx);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ AtsServerContext* server_context = ctx->server_context;
+ AtsRewriteDriverFactory* factory = (AtsRewriteDriverFactory*)server_context->factory();
+ GoogleString output;
+ StringWriter writer(&output);
+ HttpStatus::Code status = HttpStatus::kOK;
+ ContentType content_type = kContentTypeHtml;
+ StringPiece cache_control = HttpAttributes::kNoCache;
+ const char* error_message = NULL;
+ StringPiece request_uri_path = ctx->gurl->PathAndLeaf();
+
+ if (ctx->gurl->PathSansQuery() == "/robots.txt") {
+ content_type = kContentTypeText;
+ writer.Write("User-agent: *\n", server_context->message_handler());
+ writer.Write("Disallow: /\n", server_context->message_handler());
+ } else if (ctx->gurl->PathSansQuery() == "/pagespeed_message") {
+ // TODO(oschaaf)... let's wait for a bit with this one.
+ } else if (ctx->gurl->PathSansQuery() == "/pagespeed_statistics" || ctx->gurl->PathSansQuery() == "/pagespeed_global_statistics") {
+ error_message = StatisticsHandler(
+ factory,
+ server_context,
+ NULL, // No SPDY-specific config in ats_pagespeed.
+ !factory->use_per_vhost_statistics() || StringCaseStartsWith(
+ request_uri_path, "/pagespeed_global_statistics"),
+ StringPiece(ctx->gurl->Query().as_string().c_str()),
+ &content_type,
+ &writer,
+ server_context->message_handler());
+ } else if (ctx->gurl->PathSansLeaf() == "/ats_speed_static/") {
+ StringPiece file_contents;
+ if (server_context->static_asset_manager()->GetAsset(
+ request_uri_path.substr(
+ strlen("/ats_speed_static/")),
+ &file_contents, &content_type, &cache_control)) {
+ file_contents.CopyToString(&output);
+ } else {
+ error_message = "Static asset not found";
+ }
+ } else if (ctx->gurl->PathSansQuery() == "/pagespeed_console") {
+ ConsoleHandler(server_context, server_context->config(), &writer, server_context->message_handler());
+ } else {
+ // Optimized resource are highly cacheable (1 year expiry)
+ // TODO(oschaaf): configuration
+ TSHttpTxnRespCacheableSet(txn, 1);
+ TSHttpTxnReqCacheableSet(txn, 1);
+
+ TSMBuffer reqp;
+ TSMLoc req_hdr_loc;
+ if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) {
+ TSError("Error TSHttpTxnClientReqGet for resource!");
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate());
+ InterceptCtx *intercept_ctx = new InterceptCtx();
+ intercept_ctx->request_ctx = ctx;
+ intercept_ctx->request_headers = new RequestHeaders();
+ copy_request_headers_to_psol(reqp, req_hdr_loc, intercept_ctx->request_headers);
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc);
+
+
+ TSContDataSet(interceptCont, intercept_ctx);
+ TSHttpTxnServerIntercept(interceptCont, txn);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ if (error_message != NULL) {
+ status = HttpStatus::kNotFound;
+ content_type = kContentTypeHtml;
+ output = error_message;
+ }
+
+ ResponseHeaders response_headers;
+ response_headers.SetStatusAndReason(status);
+ response_headers.set_major_version(1);
+ response_headers.set_minor_version(0);
+
+ response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
+
+ int64 now_ms = factory->timer()->NowMs();
+ response_headers.SetDate(now_ms);
+ response_headers.SetLastModified(now_ms);
+ response_headers.Add(HttpAttributes::kCacheControl, cache_control);
+
+ if (FindIgnoreCase(cache_control, "private") ==
+ static_cast<int>(StringPiece::npos)) {
+ response_headers.Add(HttpAttributes::kEtag, "W/\"0\"");
+ }
+
+ GoogleString header;
+ StringWriter header_writer(&header);
+ response_headers.WriteAsHttp(&header_writer, server_context->message_handler());
+
+ TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate());
+ InterceptCtx *intercept_ctx = new InterceptCtx();
+ intercept_ctx->request_ctx = ctx;
+ header.append(output);
+ TSHttpTxnRespCacheableSet(txn, 0);
+ TSHttpTxnReqCacheableSet(txn, 0);
+ TSContDataSet(interceptCont, intercept_ctx);
+ TSHttpTxnServerIntercept(interceptCont, txn);
+ intercept_ctx->response->append(header);
+
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+}
+
+void setup_resource_intercept()
+{
+ TSCont cont = TSContCreate(read_cache_header_callback, NULL);
+ TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont);
+}
+
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_resource_intercept.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_resource_intercept.h b/plugins/experimental/ats_speed/ats_resource_intercept.h
new file mode 100644
index 0000000..31f4208
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_resource_intercept.h
@@ -0,0 +1,21 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_RESOURCE_INTERCEPT_H
+#define ATS_RESOURCE_INTERCEPT_H
+
+void setup_resource_intercept();
+
+#endif // ATS_INTERCEPT_H
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc
new file mode 100644
index 0000000..d04502d
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc
@@ -0,0 +1,178 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_rewrite_driver_factory.h"
+
+#include <cstdio>
+#include <vector>
+
+#include "ats_thread_system.h"
+#include "ats_message_handler.h"
+#include "ats_server_context.h"
+
+#include "net/instaweb/http/public/content_type.h"
+#include "net/instaweb/http/public/rate_controller.h"
+#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
+#include "net/instaweb/http/public/wget_url_fetcher.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
+#include "net/instaweb/rewriter/public/server_context.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/in_place_resource_recorder.h"
+#include "net/instaweb/system/public/serf_url_async_fetcher.h"
+#include "net/instaweb/system/public/system_caches.h"
+#include "net/instaweb/system/public/system_rewrite_options.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/null_shared_mem.h"
+#include "net/instaweb/util/public/posix_timer.h"
+#include "net/instaweb/util/public/property_cache.h"
+#include "net/instaweb/util/public/pthread_shared_mem.h"
+#include "net/instaweb/util/public/scheduler_thread.h"
+#include "net/instaweb/util/public/shared_circular_buffer.h"
+#include "net/instaweb/util/public/shared_mem_statistics.h"
+#include "net/instaweb/util/public/slow_worker.h"
+#include "net/instaweb/util/public/stdio_file_system.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
+
+
+namespace net_instaweb {
+
+
+ AtsRewriteDriverFactory::AtsRewriteDriverFactory(AtsThreadSystem* thread_system)
+ // TODO(oschaaf): fix hostname/port (?)
+ : SystemRewriteDriverFactory(thread_system, new PthreadSharedMem(), StringPiece("foohost"), 8080)
+ , ats_message_handler_(new AtsMessageHandler(thread_system->NewMutex()))
+ , ats_html_parse_message_handler_(new AtsMessageHandler(thread_system->NewMutex()))
+ , use_per_vhost_statistics_(false)
+ , threads_started_(false)
+ {
+ InitializeDefaultOptions();
+ default_options()->set_beacon_url("/ats_speed_beacon");
+ default_options()->set_enabled(RewriteOptions::kEnabledOn);
+ default_options()->SetRewriteLevel(RewriteOptions::kCoreFilters);
+
+ SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
+ default_options());
+ system_options->set_log_dir("/tmp/ps_log/");
+ system_options->set_statistics_logging_enabled(true);
+
+ system_options->set_file_cache_clean_inode_limit(500000);
+ system_options->set_file_cache_clean_size_kb(1024*10000);// 10 GB
+ system_options->set_avoid_renaming_introspective_javascript(true);
+ system_options->set_file_cache_path("/tmp/ats_ps/");
+ system_options->set_lru_cache_byte_limit(163840);
+ system_options->set_lru_cache_kb_per_process(1024*500);//500 MB
+
+ system_options->set_flush_html(true);
+
+ AtsRewriteOptions* ats_options = (AtsRewriteOptions*)system_options;
+ std::vector<std::string> args;
+ args.push_back("RateLimitBackgroundFetches");
+ args.push_back("on");
+ global_settings settings;
+ const char* msg = ats_options->ParseAndSetOptions(args, ats_message_handler_, settings);
+ CHECK(!msg);
+
+ set_message_buffer_size(1024*128);
+ set_message_handler(ats_message_handler_);
+ set_html_parse_message_handler(ats_html_parse_message_handler_);
+ StartThreads();
+ }
+
+ AtsRewriteDriverFactory::~AtsRewriteDriverFactory() {
+ ShutDown();
+ delete ats_message_handler_;
+ ats_message_handler_ = NULL;
+ delete ats_html_parse_message_handler_;
+ ats_html_parse_message_handler_ = NULL;
+ STLDeleteElements(&uninitialized_server_contexts_);
+ }
+
+ void AtsRewriteDriverFactory::InitStaticAssetManager(StaticAssetManager* static_js_manager) {
+ static_js_manager->set_library_url_prefix("/ats_speed_static/");
+ }
+
+ Hasher* AtsRewriteDriverFactory::NewHasher() {
+ return new MD5Hasher;
+ }
+
+ MessageHandler* AtsRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
+ return ats_html_parse_message_handler_;
+ }
+
+ MessageHandler* AtsRewriteDriverFactory::DefaultMessageHandler() {
+ return ats_message_handler_;
+ }
+
+ FileSystem* AtsRewriteDriverFactory::DefaultFileSystem() {
+ return new StdioFileSystem();
+ }
+
+ Timer* AtsRewriteDriverFactory::DefaultTimer() {
+ return new PosixTimer;
+ }
+
+ NamedLockManager* AtsRewriteDriverFactory::DefaultLockManager() {
+ CHECK(false) << "default lock manager should not be called";
+ return NULL;
+ }
+
+ RewriteOptions* AtsRewriteDriverFactory::NewRewriteOptions() {
+ AtsRewriteOptions* options = new AtsRewriteOptions(thread_system());
+ options->SetRewriteLevel(RewriteOptions::kCoreFilters);
+ return options;
+ }
+
+ void AtsRewriteDriverFactory::InitStats(Statistics* statistics) {
+ // Init standard PSOL stats.
+ SystemRewriteDriverFactory::InitStats(statistics);
+ // Init Ats-specific stats.
+ AtsServerContext::InitStats(statistics);
+ }
+
+
+ AtsServerContext* AtsRewriteDriverFactory::MakeAtsServerContext() {
+ AtsServerContext* server_context = new AtsServerContext(this);
+ uninitialized_server_contexts_.insert(server_context);
+ return server_context;
+ }
+
+ ServerContext* AtsRewriteDriverFactory::NewServerContext() {
+ LOG(DFATAL) << "MakeAtsServerContext should be used instead";
+ return NULL;
+ }
+
+net_instaweb::QueuedWorkerPool* AtsRewriteDriverFactory::CreateWorkerPool(net_instaweb::RewriteDriverFactory::WorkerPoolCategory pool,
+ StringPiece name) {
+ int tc = 8;
+ TSDebug("ats_speed", "Created new QueuedWorkerPool of type %d named '%s' of size %d", pool, name.data(), tc);
+ net_instaweb::QueuedWorkerPool *q_pool = new net_instaweb::QueuedWorkerPool(tc, name, thread_system());
+ return q_pool;
+}
+
+void AtsRewriteDriverFactory::StartThreads() {
+ if (threads_started_) {
+ CHECK(false) << "threads already started";
+ }
+ SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
+ bool ok = thread->Start();
+ CHECK(ok) << "Unable to start scheduler thread";
+ defer_cleanup(thread->MakeDeleter());
+ threads_started_ = true;
+}
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h
new file mode 100644
index 0000000..800cd34
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h
@@ -0,0 +1,102 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_REWRITE_DRIVER_FACTORY_H_
+#define ATS_REWRITE_DRIVER_FACTORY_H_
+
+#include <set>
+
+#include "net/instaweb/system/public/system_rewrite_driver_factory.h"
+#include "net/instaweb/util/public/md5_hasher.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
+
+
+namespace net_instaweb {
+
+
+ class AbstractSharedMem;
+ //class NgxMessageHandler;
+ //class NgxRewriteOptions;
+ class AtsServerContext;
+ class AtsThreadSystem;
+ class GoogleMessageHandler;
+ //class NgxUrlAsyncFetcher;
+ class SharedCircularBuffer;
+ class SharedMemRefererStatistics;
+ class SharedMemStatistics;
+ class SlowWorker;
+ class StaticAssetManager;
+ class Statistics;
+ class StaticAssetManager;
+ //class SystemCaches;
+
+class AtsRewriteDriverFactory : public SystemRewriteDriverFactory {
+ public:
+ explicit AtsRewriteDriverFactory(AtsThreadSystem* thread_system);
+ virtual ~AtsRewriteDriverFactory();
+
+ virtual Hasher* NewHasher();
+ virtual MessageHandler* DefaultHtmlParseMessageHandler();
+ virtual MessageHandler* DefaultMessageHandler();
+ virtual FileSystem* DefaultFileSystem();
+ virtual Timer* DefaultTimer();
+ virtual NamedLockManager* DefaultLockManager();
+ virtual RewriteOptions* NewRewriteOptions();
+
+ virtual bool UseBeaconResultsInFilters() const {
+ return true;
+ }
+
+ virtual void InitStaticAssetManager(StaticAssetManager* static_js_manager);
+
+ // Initializes all the statistics objects created transitively by
+ // AtsRewriteDriverFactory, including nginx-specific and
+ // platform-independent statistics.
+ static void InitStats(Statistics* statistics);
+
+ virtual net_instaweb::QueuedWorkerPool* CreateWorkerPool(WorkerPoolCategory pool,
+ StringPiece name);
+ virtual void NonStaticInitStats(Statistics* statistics) {
+ InitStats(statistics);
+ }
+
+ AtsServerContext* MakeAtsServerContext();
+ ServerContext* NewServerContext();
+ //AbstractSharedMem* shared_mem_runtime() const {
+ // return shared_mem_runtime_.get();
+ //}
+
+ // Starts pagespeed threads if they've not been started already. Must be
+ // called after the caller has finished any forking it intends to do.
+ void StartThreads();
+ bool use_per_vhost_statistics() const {
+ return use_per_vhost_statistics_;
+ }
+ void set_use_per_vhost_statistics(bool x) {
+ use_per_vhost_statistics_ = x;
+ }
+
+ protected:
+ private:
+ //scoped_ptr<AbstractSharedMem> shared_mem_runtime_;
+ GoogleMessageHandler* ats_message_handler_;
+ GoogleMessageHandler* ats_html_parse_message_handler_;
+ bool use_per_vhost_statistics_;
+ bool threads_started_;
+};
+
+} // namespace net_instaweb
+
+#endif // ATS_REWRITE_DRIVER_FACTORY_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_options.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_rewrite_options.cc b/plugins/experimental/ats_speed/ats_rewrite_options.cc
new file mode 100644
index 0000000..cb4f3c1
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_rewrite_options.cc
@@ -0,0 +1,256 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_rewrite_options.h"
+
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/util/public/timer.h"
+
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/rewriter/public/file_load_policy.h"
+
+#include "net/instaweb/util/public/stdio_file_system.h"
+
+#include "ats_message_handler.h"
+#include "ats_rewrite_driver_factory.h"
+
+using namespace std;
+
+namespace net_instaweb {
+
+
+RewriteOptions::Properties* AtsRewriteOptions::ats_properties_ = NULL;
+
+AtsRewriteOptions::AtsRewriteOptions(ThreadSystem* thread_system)
+ : SystemRewriteOptions(thread_system) {
+
+ Init();
+}
+
+void AtsRewriteOptions::Init() {
+ DCHECK(ats_properties_ != NULL)
+ << "Call AtsRewriteOptions::Initialize() before construction";
+ InitializeOptions(ats_properties_);
+ statistics_handler_path_.set_default("/pagespeed_global_statistics");
+}
+
+void AtsRewriteOptions::AddProperties() {
+ MergeSubclassProperties(ats_properties_);
+ AtsRewriteOptions dummy_config(NULL);
+
+ dummy_config.set_default_x_header_value(MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+}
+
+void AtsRewriteOptions::Initialize() {
+ if (Properties::Initialize(&ats_properties_)) {
+ SystemRewriteOptions::Initialize();
+ AddProperties();
+ }
+}
+
+void AtsRewriteOptions::Terminate() {
+ if (Properties::Terminate(&ats_properties_)) {
+ SystemRewriteOptions::Terminate();
+ }
+}
+
+bool AtsRewriteOptions::IsDirective(StringPiece config_directive,
+ StringPiece compare_directive) {
+ return StringCaseEqual(config_directive, compare_directive);
+}
+
+RewriteOptions::OptionSettingResult AtsRewriteOptions::ParseAndSetOptions0(
+ StringPiece directive, GoogleString* msg, MessageHandler* handler) {
+ if (IsDirective(directive, "on")) {
+ set_enabled(RewriteOptions::kEnabledOn);
+ } else if (IsDirective(directive, "off")) {
+ set_enabled(RewriteOptions::kEnabledOff);
+ } else if (IsDirective(directive, "unplugged")) {
+ set_enabled(RewriteOptions::kEnabledUnplugged);
+ } else {
+ return RewriteOptions::kOptionNameUnknown;
+ }
+ return RewriteOptions::kOptionOk;
+}
+
+
+RewriteOptions::OptionSettingResult
+AtsRewriteOptions::ParseAndSetOptionFromName1(
+ StringPiece name, StringPiece arg,
+ GoogleString* msg, MessageHandler* handler) {
+ // FileCachePath needs error checking.
+ if (StringCaseEqual(name, kFileCachePath)) {
+ if (!StringCaseStartsWith(arg, "/")) {
+ *msg = "must start with a slash";
+ return RewriteOptions::kOptionValueInvalid;
+ }
+ }
+
+ return SystemRewriteOptions::ParseAndSetOptionFromName1(
+ name, arg, msg, handler);
+}
+
+bool AtsRewriteOptions::SetBoolFlag(bool* v, StringPiece arg) {
+ if (IsDirective(arg, "on")) {
+ *v=true;
+ return true;
+ } else if (IsDirective(arg, "off")) {
+ *v=false;
+ return true;
+ }
+ return false;
+}
+
+const char*
+AtsRewriteOptions::ParseAndSetOptions(
+ vector<string> args, MessageHandler* handler, global_settings& global_config) {
+ int n_args = args.size();
+ CHECK_GE(n_args, 1);
+
+ StringPiece directive = args[0];
+
+ // Remove initial "ModPagespeed" if there is one.
+ StringPiece mod_pagespeed("ModPagespeed");
+ if (StringCaseStartsWith(directive, mod_pagespeed)) {
+ directive.remove_prefix(mod_pagespeed.size());
+ }
+
+ GoogleString msg;
+ OptionSettingResult result;
+ if (n_args == 1) {
+ result = ParseAndSetOptions0(directive, &msg, handler);
+ } else if (n_args == 2) {
+ StringPiece arg = args[1];
+ if (IsDirective(directive, "UsePerVHostStatistics")) {
+ if (!SetBoolFlag(&global_config.use_per_vhost_statistics,arg)) {
+ msg = "Failed to set UsePerVHostStatistics value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } /* else if (IsDirective(directive, "InstallCrashHandler")) {
+ // Not applicable
+ } */ else if (IsDirective(directive, "MessageBufferSize")) {
+ int message_buffer_size;
+ bool ok = StringToInt(arg.as_string(), &message_buffer_size);
+ if (ok && message_buffer_size >= 0) {
+ global_config.message_buffer_size = message_buffer_size;
+ result = RewriteOptions::kOptionOk;
+ } else {
+ msg = "Failed to set MessageBufferSize value";
+ result = RewriteOptions::kOptionValueInvalid;
+ }
+ } else if (IsDirective(directive, "UseNativeFetcher")) {
+ if (!SetBoolFlag(&global_config.info_urls_local_only,arg)) {
+ msg = "Failed to set UseNativeFetcher value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ msg = "Native fetcher is not available in this release";
+
+ result = RewriteOptions::kOptionValueInvalid;
+ }
+ } else if (IsDirective(directive, "InfoUrlsLocalOnly")) {
+ if (!SetBoolFlag(&global_config.info_urls_local_only, arg)) {
+ msg = "Failed to set InfoUrlsLocalOnly value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ }/* else if (IsDirective(directive, "RateLimitBackgroundFetches")) {
+ if (!SetBoolFlag(&global_config.rate_limit_background_fetches, arg)) {
+ msg = "Failed to set RateLimitBackgroundFetches value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } else if (IsDirective(directive, "ForceCaching")) {
+ if (!SetBoolFlag(&global_config.force_caching, arg)) {
+ msg = "Failed to set ForceCaching value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) {
+ if (!SetBoolFlag(&global_config.list_outstanding_urls_on_error, arg)) {
+ msg = "Failed to set ListOutstandingUrlsOnError value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } else if (IsDirective(directive, "TrackOriginalContentLength")) {
+ if (!SetBoolFlag(&global_config.track_original_content_length, arg)) {
+ msg = "Failed to set TrackOriginalContentLength value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } */else {
+ result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
+ }
+ } else if (n_args == 3) {
+ if (StringCaseEqual(directive, "CreateSharedMemoryMetadataCache")) {
+ int64 kb = 0;
+ if (!StringToInt64(args[2], &kb) || kb < 0) {
+ result = RewriteOptions::kOptionValueInvalid;
+ msg = "size_kb must be a positive 64-bit integer";
+ } else {
+ global_config.shm_cache_size_kb = kb;
+ result = kOptionOk;
+ //bool ok = driver_factory->caches()->CreateShmMetadataCache(
+ // args[1].as_string(), kb, &msg);
+ //result = ok ? kOptionOk : kOptionValueInvalid;
+ }
+ } else {
+ result = ParseAndSetOptionFromName2(directive, args[1], args[2],
+ &msg, handler);
+ }
+ } else if (n_args == 4) {
+ result = ParseAndSetOptionFromName3(
+ directive, args[1], args[2], args[3], &msg, handler);
+ } else {
+ return "unknown option";
+ }
+
+ if (msg.size()) {
+ handler->Message(kWarning, "Error handling config line [%s]: [%s]",
+ JoinString(args, ' ').c_str(), msg.c_str());
+ }
+
+ switch (result) {
+ case RewriteOptions::kOptionOk:
+ return NULL;
+ case RewriteOptions::kOptionNameUnknown:
+ handler->Message(kWarning, "%s", JoinString(args, ' ').c_str());
+ return "unknown option";
+ case RewriteOptions::kOptionValueInvalid: {
+ handler->Message(kWarning, "%s", JoinString(args, ' ').c_str());
+ return "Invalid value";
+ }
+ }
+
+ CHECK(false);
+ return NULL;
+}
+
+AtsRewriteOptions* AtsRewriteOptions::Clone() const {
+ AtsRewriteOptions* options = new AtsRewriteOptions(this->thread_system());
+ options->Merge(*this);
+ return options;
+}
+
+
+} // namespace net_instaweb
+
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_options.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_rewrite_options.h b/plugins/experimental/ats_speed/ats_rewrite_options.h
new file mode 100644
index 0000000..f7cfde3
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_rewrite_options.h
@@ -0,0 +1,95 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_REWRITE_OPTIONS_H_
+#define ATS_REWRITE_OPTIONS_H_
+
+#include <string>
+#include <vector>
+
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/system/public/system_rewrite_options.h"
+
+
+//#include "ats_configuration.h"
+
+
+namespace net_instaweb {
+
+class ThreadSystem;
+
+struct global_settings {
+ global_settings()
+ : info_urls_local_only(true)
+ , use_native_fetcher(false)
+ , use_per_vhost_statistics(true)
+ , message_buffer_size(1024*128)
+ , shm_cache_size_kb(0)
+ //, rate_limit_background_fetches(true)
+ //, force_caching(false)
+ //, list_outstanding_urls_on_error(false)
+ //, track_original_content_length(false)
+ {
+ }
+ bool info_urls_local_only;
+ bool use_native_fetcher;
+ bool use_per_vhost_statistics;
+ int message_buffer_size;
+ //bool rate_limit_background_fetches;
+ //bool force_caching;
+ //bool list_outstanding_urls_on_error;
+ //bool track_original_content_length;
+ int shm_cache_size_kb;
+};
+
+
+class AtsRewriteOptions : public SystemRewriteOptions {
+ public:
+ // See rewrite_options::Initialize and ::Terminate
+ static void Initialize();
+ static void Terminate();
+
+ AtsRewriteOptions(ThreadSystem* thread_system);
+ virtual ~AtsRewriteOptions() {
+ }
+
+ const char* ParseAndSetOptions(
+ std::vector<std::string> args, MessageHandler* handler, global_settings& global_config);
+
+ virtual AtsRewriteOptions* Clone() const;
+ OptionSettingResult ParseAndSetOptions0(
+ StringPiece directive, GoogleString* msg, MessageHandler* handler);
+
+ virtual OptionSettingResult ParseAndSetOptionFromName1(
+ StringPiece name, StringPiece arg,
+ GoogleString* msg, MessageHandler* handler);
+
+
+ private:
+ bool SetBoolFlag(bool* v, StringPiece arg);
+ static Properties* ats_properties_;
+ static void AddProperties();
+ void Init();
+
+ bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
+
+ DISALLOW_COPY_AND_ASSIGN(AtsRewriteOptions);
+ };
+
+} // namespace net_instaweb
+
+#endif // ATS_REWRITE_OPTIONS_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_server_context.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_server_context.cc b/plugins/experimental/ats_speed/ats_server_context.cc
new file mode 100644
index 0000000..91c3bb7
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_server_context.cc
@@ -0,0 +1,38 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#include "ats_server_context.h"
+#include "ats_rewrite_driver_factory.h"
+#include "ats_rewrite_options.h"
+
+#include "net/instaweb/system/public/system_caches.h"
+
+
+using namespace net_instaweb;
+
+AtsRewriteOptions* AtsServerContext::config() {
+ return (AtsRewriteOptions*)global_options();
+}
+
+AtsServerContext::AtsServerContext(AtsRewriteDriverFactory* factory) :
+ // TODO(oschaaf): host/port
+ SystemServerContext(factory, "foo.com" /*hostname*/, 8080/*port*/),
+ initialized_(false),
+ ats_factory_(factory) {
+}
+
+AtsServerContext::~AtsServerContext() {
+
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_server_context.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_server_context.h b/plugins/experimental/ats_speed/ats_server_context.h
new file mode 100644
index 0000000..51c5e6b
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_server_context.h
@@ -0,0 +1,48 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_SERVER_CONTEXT_H_
+#define ATS_SERVER_CONTEXT_H_
+
+#include "ats_rewrite_options.h"
+
+#include "net/instaweb/system/public/system_server_context.h"
+#include "net/instaweb/util/public/statistics.h"
+
+namespace net_instaweb {
+
+class AtsRewriteOptions;
+class AtsRewriteDriverFactory;
+
+class AtsServerContext : public net_instaweb::SystemServerContext {
+ public:
+ explicit AtsServerContext(AtsRewriteDriverFactory* factory);
+ virtual ~AtsServerContext();
+
+ virtual bool ProxiesHtml() const {
+ return true;
+ }
+
+ AtsRewriteOptions *config();
+ AtsRewriteDriverFactory *ats_rewrite_driver_factory() { return ats_factory_; }
+
+ private:
+ bool initialized_;
+ AtsRewriteDriverFactory* ats_factory_;
+};
+
+} /* ats_pagespeed */
+
+#endif /* ATS_SERVER_CONTEXT_H_ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_speed.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_speed.cc b/plugins/experimental/ats_speed/ats_speed.cc
new file mode 100644
index 0000000..afefa37
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_speed.cc
@@ -0,0 +1,1083 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+// TODO(oschaaf): remove what isn't used
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+#include <limits.h>
+#include <stdint.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+
+#include <ts/ts.h>
+
+#include <vector>
+#include <set>
+
+
+#include "ats_speed.h"
+
+#include "ats_config.h"
+#include "ats_demo_filter.h"
+#include "ats_header_utils.h"
+#include "ats_rewrite_options.h"
+#include "ats_log_message_handler.h"
+
+#include "base/logging.h"
+#include "net/instaweb/http/public/response_headers.h"
+#include "net/instaweb/util/public/string_util.h"
+
+#include "ats_base_fetch.h"
+#include "ats_resource_intercept.h"
+#include "ats_beacon_intercept.h"
+#include "ats_process_context.h"
+#include "ats_rewrite_driver_factory.h"
+#include "ats_rewrite_options.h"
+#include "ats_server_context.h"
+
+#include "net/instaweb/rewriter/public/rewrite_stats.h"
+#include "net/instaweb/system/public/in_place_resource_recorder.h"
+
+#include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/http/public/content_type.h"
+#include "net/instaweb/http/public/request_context.h"
+#include "net/instaweb/rewriter/public/experiment_matcher.h"
+#include "net/instaweb/rewriter/public/experiment_util.h"
+#include "net/instaweb/rewriter/public/process_context.h"
+#include "net/instaweb/rewriter/public/resource_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/handlers.h"
+#include "net/instaweb/public/global_constants.h"
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/google_url.h"
+#include "net/instaweb/util/public/gzip_inflater.h"
+#include "net/instaweb/util/public/query_params.h"
+#include "net/instaweb/util/public/statistics_logger.h"
+#include "net/instaweb/util/public/stdio_file_system.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_writer.h"
+#include "net/instaweb/util/public/time_util.h"
+#include "net/instaweb/util/stack_buffer.h"
+#include "net/instaweb/system/public/system_request_context.h"
+
+
+#include <dirent.h>
+
+using namespace net_instaweb;
+
+static AtsProcessContext* ats_process_context;
+static const char* DEBUG_TAG = "ats_speed_transform";
+static int TXN_INDEX_ARG;
+static int TXN_INDEX_OWNED_ARG;
+static int TXN_INDEX_OWNED_ARG_SET;
+static int TXN_INDEX_OWNED_ARG_UNSET;
+TSMutex config_mutex = TSMutexCreate();
+AtsConfig* config = NULL;
+TransformCtx* get_transaction_context(TSHttpTxn txnp) {
+ return (TransformCtx *) TSHttpTxnArgGet(txnp, TXN_INDEX_ARG);
+}
+
+static TransformCtx *
+ats_ctx_alloc()
+{
+ TransformCtx *ctx;
+
+ ctx = (TransformCtx *) TSmalloc(sizeof(TransformCtx));
+ ctx->downstream_vio = NULL;
+ ctx->downstream_buffer = NULL;
+ ctx->downstream_length = 0;
+ ctx->state = transform_state_initialized;
+
+ ctx->base_fetch = NULL;
+ ctx->proxy_fetch = NULL;
+
+ ctx->inflater = NULL;
+ ctx->url_string = NULL;
+ ctx->gurl = NULL;
+ ctx->write_pending = false;
+ ctx->fetch_done = false;
+ ctx->resource_request = false;
+ ctx->beacon_request = false;
+ ctx->transform_added = false;
+ ctx->mps_user_agent = false;
+ ctx->user_agent = NULL;
+ ctx->server_context = NULL;
+ ctx->html_rewrite = false;
+ ctx->request_method = NULL;
+ ctx->alive = 0xaaaa;
+ ctx->options = NULL;
+ ctx->to_host = NULL;
+ return ctx;
+}
+
+void
+ats_ctx_destroy(TransformCtx * ctx)
+{
+ TSReleaseAssert(ctx);
+ CHECK(ctx->alive == 0xaaaa) << "Already dead!";
+ ctx->alive = 0xbbbb;
+
+ if (ctx->base_fetch != NULL) {
+ ctx->base_fetch->Release();
+ ctx->base_fetch = NULL;
+ }
+
+ if (ctx->proxy_fetch != NULL) {
+ ctx->proxy_fetch->Done(false /* failure */);
+ ctx->proxy_fetch = NULL;
+ }
+
+ if (ctx->inflater != NULL) {
+ delete ctx->inflater;
+ ctx->inflater = NULL;
+ }
+
+ if (ctx->downstream_buffer) {
+ TSIOBufferDestroy(ctx->downstream_buffer);
+ }
+
+ if (ctx->url_string != NULL) {
+ delete ctx->url_string;
+ ctx->url_string = NULL;
+ }
+
+ if (ctx->gurl != NULL) {
+ delete ctx->gurl;
+ ctx->gurl = NULL;
+ }
+ if (ctx->user_agent != NULL) {
+ delete ctx->user_agent;
+ ctx->user_agent = NULL;
+ }
+ ctx->request_method = NULL;
+ if (ctx->options != NULL) {
+ delete ctx->options;
+ ctx->options = NULL;
+ }
+ if (ctx->to_host != NULL) {
+ delete ctx->to_host;
+ ctx->to_host = NULL;
+ }
+ TSfree(ctx);
+}
+
+RewriteOptions* ps_determine_request_options(
+ ServerContext* server_context,
+ RequestHeaders* request_headers,
+ ResponseHeaders* response_headers,
+ GoogleUrl* url) {
+ // Stripping ModPagespeed query params before the property cache lookup to
+ // make cache key consistent for both lookup and storing in cache.
+ //
+ // Sets option from request headers and url.
+ ServerContext::OptionsBoolPair query_options_success =
+ server_context->GetQueryOptions(url, request_headers,
+ response_headers);
+ bool get_query_options_success = query_options_success.second;
+ if (!get_query_options_success) {
+ // Failed to parse query params or request headers. Treat this as if there
+ // were no query params given.
+ TSError("ps_route rerquest: parsing headers or query params failed.");
+ return NULL;
+ }
+
+ // Will be NULL if there aren't any options set with query params or in
+ // headers.
+ return query_options_success.first;
+}
+
+bool ps_determine_options(ServerContext* server_context,
+ // Directory-specific options, usually null. They've already been rebased off
+ // of the global options as part of the configuration process.
+ RewriteOptions* directory_options,
+ RequestHeaders* request_headers,
+ ResponseHeaders* response_headers,
+ RewriteOptions** options,
+ GoogleUrl* url) {
+ // Global options for this server. Never null.
+ RewriteOptions* global_options = server_context->global_options();
+
+ // Request-specific options, nearly always null. If set they need to be
+ // rebased on the directory options or the global options.
+ RewriteOptions* request_options = ps_determine_request_options(
+ server_context, request_headers, response_headers, url);
+
+ // Because the caller takes ownership of any options we return, the only
+ // situation in which we can avoid allocating a new RewriteOptions is if the
+ // global options are ok as are.
+ if (directory_options == NULL && request_options == NULL &&
+ !global_options->running_experiment()) {
+ return true;
+ }
+
+ // Start with directory options if we have them, otherwise request options.
+ if (directory_options != NULL) {
+ //*options = directory_options->Clone();
+ // OS: HACK! TODO!
+ *options = global_options->Clone();
+ (*options)->Merge(*directory_options);
+ } else {
+ *options = global_options->Clone();
+ }
+
+ // Modify our options in response to request options or experiment settings,
+ // if we need to. If there are request options then ignore the experiment
+ // because we don't want experiments to be contaminated with unexpected
+ // settings.
+ if (request_options != NULL) {
+ (*options)->Merge(*request_options);
+ delete request_options;
+ }
+ // TODO(oschaaf): experiments
+ /*else if ((*options)->running_experiment()) {
+ bool ok = ps_set_experiment_state_and_cookie(
+ r, request_headers, *options, url->Host());
+ if (!ok) {
+ delete *options;
+ *options = NULL;
+ return false;
+ }
+ }*/
+
+ return true;
+}
+
+void
+handle_send_response_headers(TSHttpTxn txnp) {
+ TransformCtx* ctx = get_transaction_context(txnp);
+ // TODO(oschaaf): Fix the response headers!!
+ bool is_owned = TSHttpTxnArgGet(txnp, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET;
+ if (!is_owned) {
+ return;
+ }
+ CHECK(ctx->alive == 0xaaaa) << "Already dead !";
+ if (ctx->html_rewrite) {
+ TSMBuffer bufp = NULL;
+ TSMLoc hdr_loc = NULL;
+ if (ctx->base_fetch == NULL) {
+ // TODO(oschaaf): figure out when this happens.
+ return;
+ }
+
+ if (TSHttpTxnClientRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) {
+ ResponseHeaders* pagespeed_headers =
+ ctx->base_fetch->response_headers();
+ for (int i = 0 ; i < pagespeed_headers->NumAttributes() ; i++) {
+ const GoogleString& name_gs = pagespeed_headers->Name(i);
+ const GoogleString& value_gs = pagespeed_headers->Value(i);
+
+ // We should avoid touching these fields, as ATS will drop keepalive when we do.
+ if ( StringCaseEqual(name_gs, "Connection") || StringCaseEqual(name_gs, "Transfer-Encoding") ) {
+ continue;
+ }
+
+ TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, name_gs.data(), name_gs.size());
+ if (field_loc != NULL) {
+ TSMimeHdrFieldValuesClear(bufp, hdr_loc, field_loc);
+ TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1,
+ value_gs.data(), value_gs.size());
+ } else if (TSMimeHdrFieldCreate(bufp, hdr_loc, &field_loc) == TS_SUCCESS) {
+ if (TSMimeHdrFieldNameSet(bufp, hdr_loc, field_loc, name_gs.data(), name_gs.size()) == TS_SUCCESS) {
+ TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1,
+ value_gs.data(), value_gs.size());
+ TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc);
+ } else {
+ CHECK(false) << "Field name set failure";
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ } else {
+ CHECK(false) << "Field create failure";
+ }
+ }
+
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ } else {
+ DCHECK(false) << "Could not get response headers?!";
+ }
+ }
+}
+
+static void
+copy_response_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, ResponseHeaders* psol_headers) {
+ int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc);
+ TSMLoc field_loc;
+ const char *name, *value;
+ int name_len, value_len;
+ GoogleString header;
+ for (int i = 0; i < n_mime_headers; ++i) {
+ field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i);
+ if (!field_loc) {
+ TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i);
+ continue;
+ }
+ name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len);
+ StringPiece s_name(name, name_len);
+ int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc);
+ for (int j = 0; j < n_field_values; ++j) {
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len);
+ if ( NULL == value || !value_len ) {
+ TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]",
+ __FUNCTION__, j, name_len, name);
+ } else {
+ StringPiece s_value(value, value_len);
+ psol_headers->Add(s_name, s_value);
+ //TSDebug(DEBUG_TAG, "Add response header [%.*s:%.*s]",name_len, name, value_len, value);
+ }
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ }
+}
+
+void
+copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, RequestHeaders* psol_headers) {
+ int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc);
+ TSMLoc field_loc;
+ const char *name, *value;
+ int name_len, value_len;
+ GoogleString header;
+ for (int i = 0; i < n_mime_headers; ++i) {
+ field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i);
+ if (!field_loc) {
+ TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i);
+ continue;
+ }
+ name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len);
+ StringPiece s_name(name, name_len);
+ int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc);
+ for (int j = 0; j < n_field_values; ++j) {
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len);
+ if ( NULL == value || !value_len ) {
+ TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]",
+ __FUNCTION__, j, name_len, name);
+ } else {
+ StringPiece s_value(value, value_len);
+ psol_headers->Add(s_name, s_value);
+ //TSDebug(DEBUG_TAG, "Add request header [%.*s:%.*s]",name_len, name, value_len, value);
+ }
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ }
+}
+
+// TODO(oschaaf): this is not sustainable when we get more
+// configuration options like this.
+bool get_override_expiry(const StringPiece& host) {
+ TSMutexLock(config_mutex);
+ AtsHostConfig* hc = config->Find(host.data(), host.size());
+ TSMutexUnlock(config_mutex);
+ return hc->override_expiry();
+}
+
+AtsRewriteOptions* get_host_options(const StringPiece& host) {
+ TSMutexLock(config_mutex);
+ AtsRewriteOptions* r = NULL;
+ AtsHostConfig* hc = config->Find(host.data(), host.size());
+ if (hc->options() != NULL) {
+ // We return a clone here to avoid having to thing about
+ // configuration reloads and outstanding options
+ r = hc->options()->Clone();
+ }
+ TSMutexUnlock(config_mutex);
+ return r;
+}
+
+std::string get_remapped_host(TSHttpTxn txn) {
+ TSMBuffer server_req_buf;
+ TSMLoc server_req_loc;
+ std::string to_host;
+ if (TSHttpTxnServerReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS
+ || TSHttpTxnCachedReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS) {
+ to_host = get_header(server_req_buf, server_req_loc, "Host");
+ TSHandleMLocRelease(server_req_buf, TS_NULL_MLOC, server_req_loc);
+ } else {
+ fprintf(stderr, "@@@@@@@ FAILED \n");
+ }
+ return to_host;
+}
+
+static void
+ats_transform_init(TSCont contp, TransformCtx * ctx)
+{
+ //prepare the downstream for transforming
+ TSVConn downstream_conn;
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+ TSMBuffer reqp;
+ TSMLoc req_hdr_loc;
+ ctx->state = transform_state_output;
+
+
+ // TODO: check cleanup flow
+ if (TSHttpTxnTransformRespGet(ctx->txn, &bufp, &hdr_loc) != TS_SUCCESS) {
+ TSError("Error TSHttpTxnTransformRespGet");
+ return;
+ }
+ if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) {
+ TSError("Error TSHttpTxnClientReqGet");
+ return;
+ }
+
+ AtsServerContext* server_context = ats_process_context->server_context();
+ if (server_context->IsPagespeedResource(*ctx->gurl)) {
+ CHECK(false) << "PageSpeed resource should not get here!";
+ }
+
+ downstream_conn = TSTransformOutputVConnGet(contp);
+ ctx->downstream_buffer = TSIOBufferCreate();
+ ctx->downstream_vio = TSVConnWrite(downstream_conn, contp, TSIOBufferReaderAlloc(ctx->downstream_buffer), INT64_MAX);
+
+ // TODO(oschaaf): fix host/ip(?)
+ SystemRequestContext* system_request_context =
+ new SystemRequestContext(server_context->thread_system()->NewMutex(),
+ server_context->timer(),
+ 80,
+ "127.0.0.1");
+
+ ctx->base_fetch = new AtsBaseFetch(server_context, RequestContextPtr(system_request_context),
+ ctx->downstream_vio, ctx->downstream_buffer, false);
+
+
+ RewriteOptions* options = NULL;
+ RequestHeaders* request_headers = new RequestHeaders();
+ ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
+ copy_request_headers_to_psol(reqp, req_hdr_loc, request_headers);
+
+ TSHttpStatus status = TSHttpHdrStatusGet(bufp, hdr_loc);
+ // TODO(oschaaf): http version
+ ctx->base_fetch->response_headers()->set_status_code(status);
+ copy_response_headers_to_psol(bufp, hdr_loc, ctx->base_fetch->response_headers());
+ ctx->base_fetch->response_headers()->ComputeCaching();
+ const char* host = ctx->gurl->HostAndPort().as_string().c_str();
+ //request_headers->Lookup1(HttpAttributes::kHost);
+ if (host != NULL && strlen(host) > 0) {
+ ctx->options = get_host_options(host);
+ }
+ bool ok = ps_determine_options(server_context,
+ ctx->options,
+ request_headers,
+ ctx->base_fetch->response_headers(),
+ &options,
+ ctx->gurl);
+
+ // Take ownership of custom_options.
+ scoped_ptr<RewriteOptions> custom_options(options);
+
+ if (!ok) {
+ TSError("Failure while determining request options for psol");
+ options = server_context->global_options();
+ } else {
+ // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
+ // parameters. Keep url_string in sync with url.
+ ctx->gurl->Spec().CopyToString(ctx->url_string);
+ }
+
+ RewriteDriver* driver;
+ if (custom_options.get() == NULL) {
+ driver = server_context->NewRewriteDriver(ctx->base_fetch->request_context());
+ } else {
+ driver = server_context->NewCustomRewriteDriver(custom_options.release(), ctx->base_fetch->request_context());
+ }
+
+ driver->SetUserAgent(ctx->user_agent->c_str());
+ driver->SetRequestHeaders(*request_headers);
+
+ bool page_callback_added = false;
+ scoped_ptr<ProxyFetchPropertyCallbackCollector>
+ property_callback(
+ ProxyFetchFactory::InitiatePropertyCacheLookup(
+ false /* is resource fetch?*/,
+ *ctx->gurl,
+ server_context,
+ options,
+ ctx->base_fetch,
+ false /* requires_blink_cohort (no longer unused) */,
+ &page_callback_added));
+
+ ctx->proxy_fetch =
+ ats_process_context->proxy_fetch_factory()->CreateNewProxyFetch(
+ *(ctx->url_string), ctx->base_fetch, driver,
+ property_callback.release(),
+ NULL /* original_content_fetch */);
+
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+}
+
+static void
+ats_transform_one(TransformCtx * ctx, TSIOBufferReader upstream_reader, int amount)
+{
+ TSIOBufferBlock downstream_blkp;
+ const char *upstream_buffer;
+ int64_t upstream_length;
+
+ while (amount > 0) {
+ downstream_blkp = TSIOBufferReaderStart(upstream_reader);
+ if (!downstream_blkp) {
+ TSError("couldn't get from IOBufferBlock");
+ return;
+ }
+
+ upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
+ if (!upstream_buffer) {
+ TSError("couldn't get from TSIOBufferBlockReadStart");
+ return;
+ }
+
+ if (upstream_length > amount) {
+ upstream_length = amount;
+ }
+
+ TSDebug("ats-speed", "transform!");
+ // TODO(oschaaf): use at least the message handler from the server conrtext here?
+ if (ctx->inflater == NULL) {
+ ctx->proxy_fetch->Write(StringPiece((char*)upstream_buffer, upstream_length), ats_process_context->message_handler());
+ } else {
+ char buf[net_instaweb::kStackBufferSize];
+
+ ctx->inflater->SetInput((char*)upstream_buffer, upstream_length);
+
+ while (ctx->inflater->HasUnconsumedInput()) {
+ int num_inflated_bytes = ctx->inflater->InflateBytes(
+ buf, net_instaweb::kStackBufferSize);
+ if (num_inflated_bytes < 0) {
+ TSError("Corrupted inflation");
+ } else if (num_inflated_bytes > 0) {
+ ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes),
+ ats_process_context->message_handler());
+ }
+ }
+ }
+ //ctx->proxy_fetch->Flush(NULL);
+ TSIOBufferReaderConsume(upstream_reader, upstream_length);
+ amount -= upstream_length;
+ }
+ // TODO(oschaaf): get the output from the base fetch, and send it downstream.
+ // This would require proper locking around the base fetch buffer
+ // We could also have a look at directly writing to the traffic server buffers
+}
+
+
+static void
+ats_transform_finish(TransformCtx * ctx)
+{
+ if (ctx->state == transform_state_output) {
+ ctx->state = transform_state_finished;
+ ctx->proxy_fetch->Done(true);
+ ctx->proxy_fetch = NULL;
+ }
+}
+
+static void
+ats_transform_do(TSCont contp)
+{
+ TSVIO upstream_vio;
+ TransformCtx *ctx;
+ int64_t upstream_todo;
+ int64_t upstream_avail;
+ int64_t downstream_bytes_written;
+
+ ctx = (TransformCtx*)TSContDataGet(contp);
+
+ if (ctx->state == transform_state_initialized) {
+ ats_transform_init(contp, ctx);
+ }
+
+ upstream_vio = TSVConnWriteVIOGet(contp);
+ downstream_bytes_written = ctx->downstream_length;
+
+ if (!TSVIOBufferGet(upstream_vio)) {
+ ats_transform_finish(ctx);
+ return;
+ }
+
+ upstream_todo = TSVIONTodoGet(upstream_vio);
+
+ if (upstream_todo > 0) {
+ upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
+
+ if (upstream_todo > upstream_avail) {
+ upstream_todo = upstream_avail;
+ }
+
+ if (upstream_todo > 0) {
+ ats_transform_one(ctx, TSVIOReaderGet(upstream_vio), upstream_todo);
+ TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
+ }
+ }
+
+ if (TSVIONTodoGet(upstream_vio) > 0) {
+ if (upstream_todo > 0) {
+ if (ctx->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(ctx->downstream_vio);
+ }
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
+ }
+ } else {
+ ats_transform_finish(ctx);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
+ }
+}
+
+
+static int
+ats_speed_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
+{
+ if (TSVConnClosedGet(contp)) {
+ //ats_ctx_destroy((TransformCtx*)TSContDataGet(contp));
+ TSContDestroy(contp);
+ return 0;
+ } else {
+ switch (event) {
+ case TS_EVENT_ERROR:{
+ fprintf(stderr, "ats speed transform event: [%d] TS EVENT ERROR?!\n", event);
+ TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
+ }
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
+ break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ ats_transform_do(contp);
+ break;
+ case TS_EVENT_IMMEDIATE:
+ ats_transform_do(contp);
+ break;
+ default:
+ DCHECK(false) << "unknown event: " << event;
+ ats_transform_do(contp);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static void
+ats_speed_transform_add(TSHttpTxn txnp)
+{
+ TransformCtx* ctx = get_transaction_context(txnp);
+ CHECK(ctx);
+ if (ctx->transform_added) { // Happens with a stale cache hit
+ return;
+ } else {
+ ctx->transform_added = true;
+ }
+
+ TSHttpTxnUntransformedRespCache(txnp, 1);
+ TSHttpTxnTransformedRespCache(txnp, 0);
+
+ TSVConn connp;
+
+ connp = TSTransformCreate(ats_speed_transform, txnp);
+ TSContDataSet(connp, ctx);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
+}
+
+// Returns true if a server intercept was set up
+// Which means we should not attempt any further transformation
+void
+handle_read_request_header(TSHttpTxn txnp) {
+ TSMBuffer reqp = NULL;
+ TSMLoc hdr_loc = NULL;
+ char *url = NULL;
+ int url_length = -1;
+
+ TransformCtx* ctx = ats_ctx_alloc();
+ ctx->txn = txnp;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_ARG, (void*) ctx);
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_SET);
+
+ if (TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc) == TS_SUCCESS) {
+ url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_length);
+ if (!url || url_length <= 0) {
+ DCHECK(false) << "Could not get url!";
+ } else {
+ std::string s_url = std::string(url,url_length);
+ GoogleUrl gurl(s_url);
+
+ ctx->url_string = new GoogleString(url, url_length);
+ ctx->gurl = new GoogleUrl(*(ctx->url_string));
+ CHECK(ctx->gurl->IsWebValid()) << "Invalid URL!";
+ const char * method;
+ int method_len;
+ method = TSHttpHdrMethodGet(reqp, hdr_loc, &method_len);
+ bool head_or_get = method == TS_HTTP_METHOD_GET || method == TS_HTTP_METHOD_HEAD;
+ ctx->request_method = method;
+ GoogleString user_agent = get_header(reqp, hdr_loc, "User-Agent");
+ ctx->user_agent = new GoogleString(user_agent);
+ ctx->server_context = ats_process_context->server_context();
+ if (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos) {
+ ctx->mps_user_agent = true;
+ }
+ if (ats_process_context->server_context()->IsPagespeedResource(gurl)) {
+ if (head_or_get && !ctx->mps_user_agent) {
+ ctx->resource_request = true;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET);
+ }
+ } else if (ctx->gurl->PathSansQuery() == "/pagespeed_message"
+ || ctx->gurl->PathSansQuery() == "/pagespeed_statistics"
+ || ctx->gurl->PathSansQuery() == "/pagespeed_global_statistics"
+ || ctx->gurl->PathSansQuery() == "/pagespeed_console"
+ || ctx->gurl->PathSansLeaf() == "/ats_speed_static/"
+ || ctx->gurl->PathSansQuery() == "/robots.txt"
+ ) {
+ ctx->resource_request = true;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET);
+ }
+ else if (StringCaseEqual(gurl.PathSansQuery() ,"/ats_speed_beacon")) {
+ ctx->beacon_request = true;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET);
+ hook_beacon_intercept(txnp);
+ }
+ TSfree((void*)url);
+ }
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc);
+ } else {
+ DCHECK(false) << "Could not get client request header\n";
+ }
+
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+}
+
+bool
+cache_hit(TSHttpTxn txnp) {
+ int obj_status;
+ if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
+ // TODO(oschaaf): log warning
+ return false;
+ }
+ return obj_status == TS_CACHE_LOOKUP_HIT_FRESH;
+}
+
+static int
+transform_plugin(TSCont contp, TSEvent event, void *edata)
+{
+ TSHttpTxn txn = (TSHttpTxn) edata;
+
+ CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR
+ || event == TS_EVENT_HTTP_SEND_REQUEST_HDR || event == TS_EVENT_HTTP_READ_REQUEST_HDR
+ || event == TS_EVENT_HTTP_TXN_CLOSE || event == TS_EVENT_HTTP_SEND_RESPONSE_HDR)
+ << "Invalid transform event";
+
+ if (event != TS_EVENT_HTTP_READ_REQUEST_HDR) {
+ // Bail if an intercept is running
+ bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET;
+ if (!is_owned) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+
+ if (event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) {
+ handle_send_response_headers(txn);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ } if (event == TS_EVENT_HTTP_TXN_CLOSE) {
+ TransformCtx* ctx = get_transaction_context(txn);
+ //if (ctx != NULL && !ctx->resource_request && !ctx->beacon_request && !ctx->html_rewrite) {
+ // For intercepted requests like beacons and resource requests, we don't own the
+ // ctx here - the interceptor does.
+
+ if (ctx != NULL) {
+ bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET;
+ if (is_owned) {
+ ats_ctx_destroy(ctx);
+ }
+ }
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ } if (event == TS_EVENT_HTTP_READ_REQUEST_HDR) {
+ handle_read_request_header(txn);
+ return 0;
+ } else if (event == TS_EVENT_HTTP_SEND_REQUEST_HDR) {
+ TSMBuffer request_header_buf = NULL;
+ TSMLoc request_header_loc = NULL;
+
+ if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) {
+ hide_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding");
+ // Turn off pagespeed optimization at the origin
+ set_header(request_header_buf, request_header_loc, "PageSpeed", "off");
+ TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc);
+ } else {
+ CHECK(false) << "Could not find server request header";
+ }
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ } else if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) {
+ TSMBuffer request_header_buf = NULL;
+ TSMLoc request_header_loc = NULL;
+
+ if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) {
+ restore_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding");
+ TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc);
+ } else {
+ CHECK(false) << "Could not find server request header";
+ }
+ }
+
+ CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR);
+
+ TransformCtx* ctx = get_transaction_context(txn);
+ if (ctx == NULL) {
+ // TODO(oschaaf): document how and when this happens.
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ std::string* to_host = new std::string();
+ to_host->append(get_remapped_host(ctx->txn));
+ ctx->to_host = to_host;
+ TSMBuffer response_header_buf = NULL;
+ TSMLoc response_header_loc = NULL;
+
+ // TODO(oschaaf): from configuration!
+ bool override_expiry = false;
+
+ const char* host = ctx->gurl->HostAndPort().as_string().c_str();
+ //request_headers->Lookup1(HttpAttributes::kHost);
+ if (host != NULL && strlen(host) > 0) {
+ override_expiry = get_override_expiry(host);
+ }
+
+
+ if (ctx->mps_user_agent && override_expiry) {
+ if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) {
+ // TODO => set cacheable.
+ unset_header(response_header_buf, response_header_loc, "Cache-Control");
+ unset_header(response_header_buf, response_header_loc, "Expires");
+ unset_header(response_header_buf, response_header_loc, "Age");
+ set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600");
+ TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc);
+ }
+ }
+ bool ok = !(ctx->resource_request || ctx->beacon_request || ctx->mps_user_agent);
+ if (!ok) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ bool have_response_header = false;
+
+ if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) {
+ have_response_header = true;
+ if (override_expiry) {
+ unset_header(response_header_buf, response_header_loc, "Cache-Control");
+ unset_header(response_header_buf, response_header_loc, "Expires");
+ unset_header(response_header_buf, response_header_loc, "Age");
+ set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600");
+ }
+ }
+ else if (TSHttpTxnCachedRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) {
+ have_response_header = true;
+ }
+ if (!have_response_header) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ if (ok) {
+ if (ctx->request_method != TS_HTTP_METHOD_GET && ctx->request_method != TS_HTTP_METHOD_HEAD
+ && ctx->request_method != TS_HTTP_METHOD_POST) {
+ ok = false;
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+
+ TSHttpStatus status = TSHttpHdrStatusGet(response_header_buf, response_header_loc);
+ if (ok) {
+ if (!(status == TS_HTTP_STATUS_OK || status == TS_HTTP_STATUS_NOT_FOUND)) {
+ ok = false;
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+ if (ok) {
+ StringPiece s_content_type = get_header(response_header_buf, response_header_loc, "Content-Type");
+ const net_instaweb::ContentType* content_type =
+ net_instaweb::MimeTypeToContentType(s_content_type);
+
+ if ((content_type == NULL || !content_type->IsHtmlLike())) {
+ ok = false;
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+
+ if (ok) {
+ StringPiece content_encoding = get_header(response_header_buf, response_header_loc, "Content-Encoding");
+ net_instaweb::GzipInflater::InflateType inflate_type;
+ bool is_encoded = false;
+
+ if (StringCaseEqual(content_encoding, "deflate")) {
+ is_encoded = true;
+ inflate_type = GzipInflater::kDeflate;
+ } else if (StringCaseEqual(content_encoding, "gzip")) {
+ is_encoded = true;
+ inflate_type = GzipInflater::kGzip;
+ }
+
+ if (is_encoded) {
+ ctx->inflater = new GzipInflater(inflate_type);
+ ctx->inflater->Init();
+ }
+ TSDebug(DEBUG_TAG, "Will optimize [%s]", ctx->url_string->c_str());
+ ctx->html_rewrite = true;
+ set_header(response_header_buf,response_header_loc,"@gzip_nocache","0");
+ ats_speed_transform_add(txn);
+ }
+
+ TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+
+ return 0;
+}
+
+bool RegisterPlugin() {
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = (char *)"ats_speed";
+ info.vendor_name = (char *)"We-Amp B.V.";
+ info.support_email = (char *)"info@we-amp.com";
+
+ if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
+ TSError("Failed to register ATSSpeed");
+ return false;
+ }
+
+ return true;
+}
+
+void cleanup_process() {
+ delete ats_process_context;
+ AtsRewriteDriverFactory::Terminate();
+ AtsRewriteOptions::Terminate();
+}
+
+static void
+process_configuration()
+{
+ AtsConfig* new_config = new AtsConfig((AtsThreadSystem*)ats_process_context->server_context()->thread_system());
+ DIR *dir;
+ struct dirent *ent;
+
+ if ((dir = opendir ("/usr/local/etc/trafficserver/psol/")) != NULL) {
+ while ((ent = readdir (dir)) != NULL) {
+ size_t len = strlen(ent->d_name);
+ if (len <= 0) continue;
+ if (ent->d_name[0] == '.') continue;
+ if (ent->d_name[len-1] == '~') continue;
+ if (ent->d_name[0] == '#') continue;
+ GoogleString s("/usr/local/etc/trafficserver/psol/");
+ s.append(ent->d_name);
+ fprintf (stderr, "parse [%s]\n", s.c_str());
+ if (!new_config->Parse(s.c_str())) {
+ TSError("Error parsing %s", s.c_str());
+ }
+ }
+ closedir (dir);
+ }
+
+ AtsConfig* old_config;
+ TSMutexLock(config_mutex);
+ fprintf(stderr, "Update configuration\n");
+ old_config = config;
+ config = new_config;
+ TSMutexUnlock(config_mutex);
+ if (old_config != NULL) {
+ delete old_config;
+ }
+}
+
+static void *
+config_notification_callback(void *data)
+{
+ int BUF_MAX = 1024 * (sizeof(struct inotify_event) + 16);
+ char buf[BUF_MAX];
+ int fd,wd;
+
+ fd = inotify_init();
+
+ if (fd < 0) {
+ perror( "inotify_init" );
+ CHECK(false) << "Failed to initialize inotify";
+ }
+
+ wd = inotify_add_watch(fd, "/usr/local/etc/trafficserver/psol/", IN_MODIFY | IN_CREATE | IN_DELETE);
+
+ while (1) {
+ int len = read(fd, buf, BUF_MAX);
+ int i = 0;
+ bool do_update = false;
+ while ( i < len ) {
+ struct inotify_event *event = ( struct inotify_event * ) &buf[ i ];
+ if ( event->len ) {
+ if (!(event->mask & IN_ISDIR)) {
+ const char* name = event->name;
+ size_t name_len = strlen(event->name);
+ if (name_len > 0 && name[0] != '.' && name[0] != '#' && name[name_len-1] != '~' ) {
+ do_update = true;
+ }
+ }
+ }
+ i += ( sizeof (struct inotify_event) ) + event->len;
+ }
+ if (do_update) {
+ process_configuration();
+ }
+ }
+
+ inotify_rm_watch( fd, wd );
+ close( fd );
+
+ return NULL;
+}
+
+
+void TSPluginInit(int argc, const char *argv[]) {
+ if (RegisterPlugin() == true) {
+ if (TSHttpArgIndexReserve("ats_speed", "Stores the transaction context", &TXN_INDEX_ARG) != TS_SUCCESS) {
+ CHECK(false) << "failed to reserve an argument index";
+ }
+ if (TSHttpArgIndexReserve("ats_speed", "Stores the transaction context", &TXN_INDEX_OWNED_ARG) != TS_SUCCESS) {
+ CHECK(false) << "failed to reserve an argument index";
+ }
+
+ AtsRewriteOptions::Initialize();
+ AtsRewriteDriverFactory::Initialize();
+ net_instaweb::log_message_handler::Install();
+ atexit(cleanup_process);
+ ats_process_context = new AtsProcessContext();
+ process_configuration();
+ TSCont transform_contp = TSContCreate(transform_plugin, NULL);
+ TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_READ_CACHE_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_SEND_RESPONSE_HDR_HOOK, transform_contp);
+
+ setup_resource_intercept();
+ CHECK(TSThreadCreate(config_notification_callback, NULL)) << "";
+ ats_process_context->message_handler()->Message(
+ kInfo, "TSPluginInit OK");
+ }
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_speed.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_speed.h b/plugins/experimental/ats_speed/ats_speed.h
new file mode 100644
index 0000000..99b9bd4
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_speed.h
@@ -0,0 +1,94 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+#ifndef ATS_SPEED_H_
+#define ATS_SPEED_H_
+
+#include <string>
+
+#include <ts/ts.h>
+
+#include "net/instaweb/util/public/google_url.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+class AtsBaseFetch;
+class AtsRewriteOptions;
+class AtsServerContext;
+class GzipInflater;
+class ProxyFetch;
+class RewriteOptions;
+class RequestHeaders;
+class ResponseHeaders;
+class ServerContext;
+
+} // namespace net_instaweb
+
+enum transform_state {
+ transform_state_initialized,
+ transform_state_output,
+ transform_state_finished
+};
+
+typedef struct
+{
+ TSHttpTxn txn;
+ TSVIO downstream_vio;
+ TSIOBuffer downstream_buffer;
+ int64_t downstream_length;
+ enum transform_state state;
+
+ net_instaweb::AtsBaseFetch* base_fetch;
+ net_instaweb::ProxyFetch* proxy_fetch;
+ net_instaweb::GzipInflater* inflater;
+
+ bool write_pending;
+ bool fetch_done;
+ GoogleString* url_string;
+ bool beacon_request;
+ bool resource_request;
+ bool mps_user_agent;
+ bool transform_added;
+ net_instaweb::GoogleUrl* gurl;
+ net_instaweb::AtsServerContext* server_context;
+ GoogleString* user_agent;
+ bool html_rewrite;
+ const char* request_method;
+ int alive;
+ net_instaweb::AtsRewriteOptions* options;
+ // TODO: Use GoogleString*
+ std::string* to_host;
+} TransformCtx;
+
+TransformCtx* get_transaction_context(TSHttpTxn txnp);
+void ats_ctx_destroy(TransformCtx * ctx);
+bool cache_hit(TSHttpTxn txnp);
+
+bool ps_determine_options(net_instaweb::ServerContext* server_context,
+ // Directory-specific options, usually null. They've already been rebased off
+ // of the global options as part of the configuration process.
+ net_instaweb::RewriteOptions* directory_options,
+ net_instaweb::RequestHeaders* request_headers,
+ net_instaweb::ResponseHeaders* response_headers,
+ net_instaweb::RewriteOptions** options,
+ net_instaweb::GoogleUrl* url);
+
+void copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, net_instaweb::RequestHeaders* psol_headers);
+// You will own options returned by this:
+net_instaweb::AtsRewriteOptions* get_host_options(const StringPiece& host);
+
+#endif /* ATS_SPEED_H_ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_thread_system.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ats_thread_system.h b/plugins/experimental/ats_speed/ats_thread_system.h
new file mode 100644
index 0000000..ebbf5e9
--- /dev/null
+++ b/plugins/experimental/ats_speed/ats_thread_system.h
@@ -0,0 +1,43 @@
+// Copyright 2013 We-Amp B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+#ifndef ATS_THREAD_SYSTEM_H_
+#define ATS_THREAD_SYSTEM_H_
+
+#include <pthread.h>
+
+#include <ts/ts.h>
+#include "net/instaweb/system/public/system_thread_system.h"
+#include "net/instaweb/util/public/thread.h"
+#include "net/instaweb/util/public/thread_system.h"
+#include "net/instaweb/util/public/pthread_rw_lock.h"
+#include "net/instaweb/util/public/condvar.h"
+
+namespace net_instaweb {
+
+class AtsThreadSystem : public net_instaweb::SystemThreadSystem {
+ public:
+ virtual void BeforeThreadRunHook() {
+ TSThreadInit();
+ }
+
+ virtual ~AtsThreadSystem() { }
+};
+
+} // net_instaweb
+
+
+#endif // ATS_THREAD_SYSTEM_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ethread.patch
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/ethread.patch b/plugins/experimental/ats_speed/ethread.patch
new file mode 100644
index 0000000..3cc5e1c
--- /dev/null
+++ b/plugins/experimental/ats_speed/ethread.patch
@@ -0,0 +1,13 @@
+diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h
+index e5f1c56..3d2f9c7 100644
+--- a/iocore/eventsystem/I_EThread.h
++++ b/iocore/eventsystem/I_EThread.h
+@@ -297,7 +297,7 @@ public:
+ virtual ~EThread();
+
+ Event *schedule_spawn(Continuation *cont);
+- Event *schedule(Event *e, bool fast_signal = false);
++ Event *schedule(Event *e, bool fast_signal = true);
+
+ /** Block of memory to allocate thread specific data e.g. stat system arrays. */
+ char thread_private[PER_THREAD_DATA];
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/Makefile
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_speed/gzip/Makefile b/plugins/experimental/ats_speed/gzip/Makefile
new file mode 100644
index 0000000..0b8d12e
--- /dev/null
+++ b/plugins/experimental/ats_speed/gzip/Makefile
@@ -0,0 +1,9 @@
+TSXS?=tsxs
+
+all:
+ $(TSXS) -o gzip.so -v -C *.cc
+install:
+ $(TSXS) -v -i -o gzip.so
+clean:
+ rm -f *.lo *.so
+