You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@trafficserver.apache.org by James Peach <jp...@apache.org> on 2014/08/04 17:25:51 UTC
Re: [1/3] ats_speed: PageSpeed optimization plugin
Great! Thanks a lot Otto :)
On Aug 4, 2014, at 6:57 AM, oschaaf@apache.org wrote:
> Repository: trafficserver
> Updated Branches:
> refs/heads/master fab202517 -> 083abd4ff
>
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/README
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/README b/plugins/experimental/ats_speed/gzip/README
> new file mode 100644
> index 0000000..2e74681
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/README
> @@ -0,0 +1,4 @@
> +This gzip is not compiled, but only here to be able to diff later
> +with the official gzip plugin. It is very slightly modified no
> +be able to disable caching of its compressed output through a
> +response header
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/configuration.cc b/plugins/experimental/ats_speed/gzip/configuration.cc
> new file mode 100644
> index 0000000..b1c499d
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/configuration.cc
> @@ -0,0 +1,264 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#include "configuration.h"
> +#include <fstream>
> +#include <algorithm>
> +#include <vector>
> +#include <fnmatch.h>
> +
> +namespace Gzip {
> + using namespace std;
> +
> + void ltrim_if(string& s, int (* fp) (int)) {
> + for (size_t i = 0; i < s.size();) {
> + if (fp(s[i])) {
> + s.erase(i,1);
> + } else {
> + break;
> + }
> + }
> + }
> +
> + void rtrim_if(string& s, int (* fp) (int)) {
> + for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) {
> + if (fp(s[i])) {
> + s.erase(i,1);
> + } else {
> + break;
> + }
> + }
> + }
> +
> + void trim_if(string& s, int (* fp) (int)) {
> + ltrim_if(s, fp);
> + rtrim_if(s, fp);
> + }
> +
> + vector<string> tokenize(const string &s, int (* fp) (int)) {
> + vector<string> r;
> + string tmp;
> +
> + for (size_t i = 0; i < s.size(); i++) {
> + if ( fp(s[i]) ) {
> + if ( tmp.size() ) {
> + r.push_back(tmp);
> + tmp = "";
> + }
> + } else {
> + tmp += s[i];
> + }
> + }
> +
> + if ( tmp.size() ) {
> + r.push_back(tmp);
> + }
> +
> + return r;
> + }
> +
> + enum ParserState {
> + kParseStart,
> + kParseCompressibleContentType,
> + kParseRemoveAcceptEncoding,
> + kParseEnable,
> + kParseCache,
> + kParseDisallow,
> + };
> +
> + void Configuration::AddHostConfiguration(HostConfiguration * hc){
> + host_configurations_.push_back(hc);
> + }
> +
> + void HostConfiguration::add_disallow(const std::string & disallow) {
> + disallows_.push_back(disallow);
> + }
> +
> + void HostConfiguration::add_compressible_content_type(const std::string & content_type) {
> + compressible_content_types_.push_back(content_type);
> + }
> +
> + HostConfiguration * Configuration::Find(const char * host, int host_length) {
> + HostConfiguration * host_configuration = host_configurations_[0];
> +
> + std::string shost(host, host_length);
> +
> + for (size_t i = 1; i < host_configurations_.size(); i++ ) {
> + if (host_configurations_[i]->host() == shost){
> + host_configuration = host_configurations_[i];
> + break;
> + }
> + }
> +
> + return host_configuration;
> + }
> +
> + bool HostConfiguration::IsUrlAllowed(const char * url, int url_len) {
> + string surl(url, url_len);
> +
> + for (size_t i = 0; i < disallows_.size(); i++) {
> + if ( fnmatch (disallows_[i].c_str(), surl.c_str(), 0) == 0 ) {
> + info("url [%s] disabled for compression, matched on pattern [%s]",
> + surl.c_str(), disallows_[i].c_str());
> + return false;
> + }
> + }
> +
> + return true;
> + }
> +
> + bool HostConfiguration::ContentTypeIsCompressible(const char * content_type, int content_type_length) {
> + string scontent_type(content_type, content_type_length);
> + bool is_match = false;
> +
> + for (size_t i = 0; i < compressible_content_types_.size(); i++) {
> + const char* match_string = compressible_content_types_[i].c_str();
> + bool exclude = match_string[0] == '!';
> + if (exclude) {
> + match_string++;//skip '!'
> + }
> + if ( fnmatch (match_string, scontent_type.c_str(), 0) == 0 ) {
> + info("compressible content type [%s], matched on pattern [%s]",
> + scontent_type.c_str(), compressible_content_types_[i].c_str());
> + is_match = !exclude;
> + }
> + }
> +
> + return is_match;
> + }
> +
> + Configuration * Configuration::Parse(const char * path ) {
> + string pathstring(path);
> +
> + // If we have a path and it's not an absolute path, make it relative to the
> + // configuration directory.
> + if (!pathstring.empty() && pathstring[0] != '/') {
> + pathstring.assign(TSConfigDirGet());
> + pathstring.append("/");
> + pathstring.append(path);
> + }
> +
> + trim_if(pathstring, isspace);
> +
> + Configuration * c = new Configuration();
> + HostConfiguration * current_host_configuration = new HostConfiguration("");
> + c->AddHostConfiguration(current_host_configuration);
> + current_host_configuration->add_compressible_content_type("text/*");
> + current_host_configuration->add_compressible_content_type("application/xml*");
> + current_host_configuration->add_compressible_content_type("*javascript*");
> + current_host_configuration->add_compressible_content_type("image/svg+xml");
> +
> +
> + if (pathstring.empty()) {
> + return c;
> + }
> +
> + path = pathstring.c_str();
> + info("Parsing file \"%s\"", path);
> + std::ifstream f;
> +
> + size_t lineno = 0;
> +
> + f.open(path, std::ios::in);
> +
> + if (!f.is_open()) {
> + warning("could not open file [%s], skip",path);
> + return c;
> + }
> +
> + enum ParserState state = kParseStart;
> +
> + while (!f.eof()) {
> + std::string line;
> + getline(f, line);
> + ++lineno;
> +
> + trim_if(line, isspace);
> + if (line.size() == 0) {
> + continue;
> + }
> +
> + vector<string> v = tokenize( line, isspace );
> +
> + for(size_t i = 0; i < v.size(); i++ ) {
> + string token = v[i];
> + trim_if(token, isspace);
> +
> + //should not happen
> + if (!token.size()) continue;
> +
> + //once a comment is encountered, we are done processing the line
> + if (token[0] == '#') break;
> +
> + switch(state) {
> + case kParseStart:
> + if ( (token[0] == '[') && (token[token.size()-1] == ']')){
> + std::string current_host = token.substr(1,token.size()-2);
> + current_host_configuration = new HostConfiguration(current_host);
> + c->AddHostConfiguration(current_host_configuration);
> + } else if (token == "compressible-content-type" ) {
> + state = kParseCompressibleContentType;
> + } else if (token == "remove-accept-encoding" ) {
> + state = kParseRemoveAcceptEncoding;
> + } else if (token == "enabled" ) {
> + state = kParseEnable;
> + } else if (token == "cache" ) {
> + state = kParseCache;
> + } else if (token == "disallow" ) {
> + state = kParseDisallow;
> + }
> + else {
> + warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno);
> + }
> + break;
> + case kParseCompressibleContentType:
> + current_host_configuration->add_compressible_content_type(token);
> + state = kParseStart;
> + break;
> + case kParseRemoveAcceptEncoding:
> + current_host_configuration->set_remove_accept_encoding(token == "true");
> + state = kParseStart;
> + break;
> + case kParseEnable:
> + current_host_configuration->set_enabled(token == "true");
> + state = kParseStart;
> + break;
> + case kParseCache:
> + current_host_configuration->set_cache(token == "true");
> + state = kParseStart;
> + break;
> + case kParseDisallow:
> + current_host_configuration->add_disallow(token);
> + state = kParseStart;
> + break;
> + }
> + }
> + }
> +
> + if (state != kParseStart) {
> + warning("the parser state indicates that data was expected when it reached the end of the file (%d)", state);
> + }
> +
> + return c;
> + } //Configuration::Parse
> +} //namespace
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/configuration.h b/plugins/experimental/ats_speed/gzip/configuration.h
> new file mode 100644
> index 0000000..b38cb64
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/configuration.h
> @@ -0,0 +1,84 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +
> +#ifndef GZIP_CONFIGURATION_H_
> +#define GZIP_CONFIGURATION_H_
> +
> +#include <string>
> +#include <vector>
> +#include "debug_macros.h"
> +
> +namespace Gzip {
> + class HostConfiguration {
> + public: //todo -> only configuration should be able to construct hostconfig
> + explicit HostConfiguration(const std::string & host)
> + : host_(host)
> + , enabled_(true)
> + , cache_(true)
> + , remove_accept_encoding_(false)
> + {}
> +
> + inline bool enabled() { return enabled_; }
> + inline void set_enabled(bool x) { enabled_ = x; }
> + inline bool cache() { return cache_; }
> + inline void set_cache(bool x) { cache_ = x; }
> + inline bool remove_accept_encoding() { return remove_accept_encoding_; }
> + inline void set_remove_accept_encoding(bool x) { remove_accept_encoding_ = x; }
> + inline std::string host() { return host_; }
> + void add_disallow(const std::string & disallow);
> + void add_compressible_content_type(const std::string & content_type);
> + bool IsUrlAllowed(const char * url, int url_len);
> + bool ContentTypeIsCompressible(const char * content_type, int content_type_length);
> +
> + private:
> + std::string host_;
> + bool enabled_;
> + bool cache_;
> + bool remove_accept_encoding_;
> + std::vector<std::string> compressible_content_types_;
> + std::vector<std::string> disallows_;
> + DISALLOW_COPY_AND_ASSIGN(HostConfiguration);
> + };//class HostConfiguration
> +
> + class Configuration {
> + friend class HostConfiguration;
> + public:
> + static Configuration * Parse(const char * path);
> + HostConfiguration * Find(const char * host, int host_length);
> + inline HostConfiguration * GlobalConfiguration() {
> + return host_configurations_[0];
> + }
> +
> + private:
> + explicit Configuration() {}
> + void AddHostConfiguration(HostConfiguration * hc);
> +
> + std::vector<HostConfiguration *> host_configurations_;
> + //todo: destructor. delete owned host configurations
> + DISALLOW_COPY_AND_ASSIGN(Configuration);
> + }; //class Configuration
> +
> +}//namespace
> +
> +#endif
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/debug_macros.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/debug_macros.h b/plugins/experimental/ats_speed/gzip/debug_macros.h
> new file mode 100644
> index 0000000..151de31
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/debug_macros.h
> @@ -0,0 +1,59 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#ifndef _DBG_MACROS_H
> +#define _DBG_MACROS_H
> +
> +#include <ts/ts.h>
> +
> +#define TAG "gzip"
> +
> +#define debug(fmt, args...) do { \
> + TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + } while (0)
> +
> +#define info(fmt, args...) do { \
> + TSDebug(TAG, "INFO: " fmt, ##args ); \
> + } while (0)
> +
> +#define warning(fmt, args...) do { \
> + TSDebug(TAG, "WARNING: " fmt, ##args ); \
> +} while (0)
> +
> +#define error(fmt, args...) do { \
> + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> +} while (0)
> +
> +#define fatal(fmt, args...) do { \
> + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
> + exit(-1); \
> +} while (0)
> +
> +//FIXME: this one doesn't deserve to be here
> +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
> + TypeName(const TypeName&); \
> + void operator=(const TypeName&)
> +
> +#endif //_DBG_MACROS_H
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/gzip.cc b/plugins/experimental/ats_speed/gzip/gzip.cc
> new file mode 100644
> index 0000000..1397762
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/gzip.cc
> @@ -0,0 +1,826 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +#define __STDC_LIMIT_MACROS
> +#define __STDC_CONSTANT_MACROS
> +#include <stdint.h>
> +
> +#include <string>
> +#include <string.h>
> +#include <zlib.h>
> +#include <ts/ts.h>
> +#include "debug_macros.h"
> +#include "misc.h"
> +#include "configuration.h"
> +//#include "ink_defs.h"
> +
> +using namespace std;
> +using namespace Gzip;
> +
> +//FIXME: custom dictionaries would be nice. configurable/content-type?
> +//FIXME: look into autoscaling the compression level based on connection speed
> +// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
> +// might be served better with little or no compression at all
> +//FIXME: look into compressing from the task thread pool
> +//FIXME: make normalizing accept encoding configurable
> +
> +// from mod_deflate:
> +// ZLIB's compression algorithm uses a
> +// 0-9 based scale that GZIP does where '1' is 'Best speed'
> +// and '9' is 'Best compression'. Testing has proved level '6'
> +// to be about the best level to use in an HTTP Server.
> +
> +const int ZLIB_COMPRESSION_LEVEL = 6;
> +
> +int arg_idx_hooked;
> +int arg_idx_host_configuration;
> +int arg_idx_url_disallowed;
> +
> +
> +const char * global_hidden_header_name;
> +Configuration* config = NULL;
> +const char *dictionary = NULL;
> +
> +static GzipData *
> +gzip_data_alloc(int compression_type)
> +{
> + GzipData *data;
> + int err;
> +
> + data = (GzipData *) TSmalloc(sizeof(GzipData));
> + data->downstream_vio = NULL;
> + data->downstream_buffer = NULL;
> + data->downstream_reader = NULL;
> + data->downstream_length = 0;
> + data->state = transform_state_initialized;
> + data->compression_type = compression_type;
> + data->zstrm.next_in = Z_NULL;
> + data->zstrm.avail_in = 0;
> + data->zstrm.total_in = 0;
> + data->zstrm.next_out = Z_NULL;
> + data->zstrm.avail_out = 0;
> + data->zstrm.total_out = 0;
> + data->zstrm.zalloc = gzip_alloc;
> + data->zstrm.zfree = gzip_free;
> + data->zstrm.opaque = (voidpf) 0;
> + data->zstrm.data_type = Z_ASCII;
> +
> + int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
> +
> + err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
> +
> + if (err != Z_OK) {
> + fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
> + }
> +
> + if (dictionary) {
> + err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
> + if (err != Z_OK) {
> + fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
> + }
> + }
> +
> + return data;
> +}
> +
> +
> +static void
> +gzip_data_destroy(GzipData * data)
> +{
> + TSReleaseAssert(data);
> +
> + //deflateEnd returnvalue ignore is intentional
> + //it would spew log on every client abort
> + deflateEnd(&data->zstrm);
> +
> + if (data->downstream_buffer) {
> + TSIOBufferDestroy(data->downstream_buffer);
> + }
> +
> + TSfree(data);
> +}
> +
> +static TSReturnCode
> +gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type)
> +{
> + TSReturnCode ret;
> + TSMLoc ce_loc;
> +
> + // Delete Content-Encoding if present???
> +
> + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Content-Encoding", sizeof("Content-Encoding") - 1, &ce_loc)) == TS_SUCCESS) {
> + if (compression_type == COMPRESSION_TYPE_DEFLATE) {
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", sizeof("deflate") - 1);
> + } else if (compression_type == COMPRESSION_TYPE_GZIP) {
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", sizeof("gzip") - 1);
> + }
> + if (ret == TS_SUCCESS) {
> + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot add the Content-Encoding header");
> + }
> +
> + return ret;
> +}
> +
> +static TSReturnCode
> +gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
> +{
> + TSReturnCode ret;
> + TSMLoc ce_loc;
> +
> + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Vary", sizeof("Vary") - 1);
> + if (ce_loc) {
> + int idx, count, len;
> + const char *value;
> +
> + count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc);
> + for(idx=0; idx<count; idx++) {
> + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len);
> + if (len &&
> + strncasecmp("Accept-Encoding", value, len) == 0) {
> + // Bail, Vary: Accept-Encoding already sent from origin
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + return TS_SUCCESS;
> + }
> + }
> +
> + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1);
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + } else {
> + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Vary", sizeof("Vary") - 1, &ce_loc)) == TS_SUCCESS) {
> + if ((ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", sizeof("Accept-Encoding") - 1)) == TS_SUCCESS) {
> + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
> + }
> +
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot add/update the Vary header");
> + }
> +
> + return ret;
> +}
> +
> +//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
> +// specify a very header..
> +static TSReturnCode
> +gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
> +{
> + TSReturnCode ret = TS_SUCCESS;
> + TSMLoc ce_loc;
> +
> + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
> +
> + if (ce_loc) {
> + int changetag = 1;
> + int strl;
> + const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
> + //do not alter weak etags.
> + //FIXME: consider just making the etag weak for compressed content
> + if (strl >= 2) {
> + if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
> + changetag = 0;
> + }
> + if (changetag) {
> + ret = TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
> + }
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
> + }
> +
> + if (ret != TS_SUCCESS) {
> + error("cannot handle the %s header", TS_MIME_FIELD_ETAG);
> + }
> +
> + return ret;
> +}
> +
> +//FIXME: some things are potentially compressible. those responses
> +static void
> +gzip_transform_init(TSCont contp, GzipData * data)
> +{
> + //update the vary, content-encoding, and etag response headers
> + //prepare the downstream for transforming
> +
> + TSVConn downstream_conn;
> + TSMBuffer bufp;
> + TSMLoc hdr_loc;
> +
> + data->state = transform_state_output;
> +
> + if (TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc) != TS_SUCCESS) {
> + error("Error TSHttpTxnTransformRespGet");
> + return;
> + }
> +
> + if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == TS_SUCCESS &&
> + gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS &&
> + gzip_etag_header(bufp, hdr_loc) == TS_SUCCESS) {
> + downstream_conn = TSTransformOutputVConnGet(contp);
> + data->downstream_buffer = TSIOBufferCreate();
> + data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
> + data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
> + }
> +
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> +}
> +
> +
> +
> +static void
> +gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
> +{
> + TSIOBufferBlock downstream_blkp;
> + const char *upstream_buffer;
> + char *downstream_buffer;
> + int64_t upstream_length, downstream_length;
> + int err;
> +
> + while (amount > 0) {
> + downstream_blkp = TSIOBufferReaderStart(upstream_reader);
> + if (!downstream_blkp) {
> + error("couldn't get from IOBufferBlock");
> + return;
> + }
> +
> + upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
> + if (!upstream_buffer) {
> + error("couldn't get from TSIOBufferBlockReadStart");
> + return;
> + }
> +
> + if (upstream_length > amount) {
> + upstream_length = amount;
> + }
> +
> + data->zstrm.next_in = (unsigned char *) upstream_buffer;
> + data->zstrm.avail_in = upstream_length;
> +
> + while (data->zstrm.avail_in > 0) {
> + downstream_blkp = TSIOBufferStart(data->downstream_buffer);
> + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
> +
> + data->zstrm.next_out = (unsigned char *) downstream_buffer;
> + data->zstrm.avail_out = downstream_length;
> +
> + err = deflate(&data->zstrm, Z_NO_FLUSH);
> +
> + if (err != Z_OK)
> + warning("deflate() call failed: %d", err);
> +
> + if (downstream_length > data->zstrm.avail_out) {
> + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
> + data->downstream_length += (downstream_length - data->zstrm.avail_out);
> + }
> +
> + if (data->zstrm.avail_out > 0) {
> + if (data->zstrm.avail_in != 0) {
> + error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
> + }
> + }
> + }
> +
> + TSIOBufferReaderConsume(upstream_reader, upstream_length);
> + amount -= upstream_length;
> + }
> +}
> +
> +static void
> +gzip_transform_finish(GzipData * data)
> +{
> + if (data->state == transform_state_output) {
> + TSIOBufferBlock downstream_blkp;
> + char *downstream_buffer;
> + int64_t downstream_length;
> + int err;
> +
> + data->state = transform_state_finished;
> +
> + for (;;) {
> + downstream_blkp = TSIOBufferStart(data->downstream_buffer);
> +
> + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
> + data->zstrm.next_out = (unsigned char *) downstream_buffer;
> + data->zstrm.avail_out = downstream_length;
> +
> + err = deflate(&data->zstrm, Z_FINISH);
> +
> + if (downstream_length > (int64_t) data->zstrm.avail_out) {
> + TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
> + data->downstream_length += (downstream_length - data->zstrm.avail_out);
> + }
> +
> + if (err == Z_OK) { /* some more data to encode */
> + continue;
> + }
> +
> + if (err != Z_STREAM_END) {
> + warning("deflate should report Z_STREAM_END");
> + }
> + break;
> + }
> +
> + if (data->downstream_length != (int64_t) (data->zstrm.total_out)) {
> + error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
> + data->zstrm.total_out);
> + }
> +
> + gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
> + }
> +}
> +
> +
> +static void
> +gzip_transform_do(TSCont contp)
> +{
> + TSVIO upstream_vio;
> + GzipData *data;
> + int64_t upstream_todo;
> + int64_t upstream_avail;
> + int64_t downstream_bytes_written;
> +
> + data = (GzipData*)TSContDataGet(contp);
> + if (data->state == transform_state_initialized) {
> + gzip_transform_init(contp, data);
> + }
> +
> + upstream_vio = TSVConnWriteVIOGet(contp);
> + downstream_bytes_written = data->downstream_length;
> +
> + if (!TSVIOBufferGet(upstream_vio)) {
> + gzip_transform_finish(data);
> +
> + TSVIONBytesSet(data->downstream_vio, data->downstream_length);
> +
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> + return;
> + }
> +
> + upstream_todo = TSVIONTodoGet(upstream_vio);
> +
> + if (upstream_todo > 0) {
> + upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
> +
> + if (upstream_todo > upstream_avail) {
> + upstream_todo = upstream_avail;
> + }
> +
> + if (upstream_todo > 0) {
> + gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
> + TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
> + }
> + }
> +
> + if (TSVIONTodoGet(upstream_vio) > 0) {
> + if (upstream_todo > 0) {
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
> + }
> + } else {
> + gzip_transform_finish(data);
> + TSVIONBytesSet(data->downstream_vio, data->downstream_length);
> +
> + if (data->downstream_length > downstream_bytes_written) {
> + TSVIOReenable(data->downstream_vio);
> + }
> +
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
> + }
> +}
> +
> +
> +static int
> +gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
> +{
> + if (TSVConnClosedGet(contp)) {
> + gzip_data_destroy((GzipData*)TSContDataGet(contp));
> + TSContDestroy(contp);
> + return 0;
> + } else {
> + switch (event) {
> + case TS_EVENT_ERROR:{
> + debug("gzip_transform: TS_EVENT_ERROR starts");
> + TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
> + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
> + }
> + break;
> + case TS_EVENT_VCONN_WRITE_COMPLETE:
> + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
> + break;
> + case TS_EVENT_VCONN_WRITE_READY:
> + gzip_transform_do(contp);
> + break;
> + case TS_EVENT_IMMEDIATE:
> + gzip_transform_do(contp);
> + break;
> + default:
> + warning("unknown event [%d]", event);
> + gzip_transform_do(contp);
> + break;
> + }
> + }
> +
> + return 0;
> +}
> +
> +
> +static int
> +gzip_transformable(TSHttpTxn txnp, int server, HostConfiguration * host_configuration, int *compress_type)
> +{
> + /* Server response header */
> + TSMBuffer bufp;
> + TSMLoc hdr_loc;
> + TSMLoc field_loc;
> +
> + /* Client request header */
> + TSMBuffer cbuf;
> + TSMLoc chdr;
> + TSMLoc cfield;
> +
> + const char *value;
> + int nvalues;
> + int i, compression_acceptable, len;
> +
> + TSHttpStatus resp_status;
> + if (server) {
> + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
> + } else {
> + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
> + }
> + resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> +
> + //conservatively pick some statusses to compress
> + if (!(resp_status == 200 || resp_status == 404 || resp_status == 500)) {
> + info("http response status [%d] is not compressible", resp_status);
> + return 0;
> + }
> +
> + TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
> +
> + //the only compressible method is currently GET.
> + int method_length;
> + const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length);
> + if (!(method_length == TS_HTTP_LEN_GET && memcmp(method, TS_HTTP_METHOD_GET, TS_HTTP_LEN_GET) == 0)) {
> + debug("method is not GET, not compressible");
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> + return 0;
> + }
> +
> + cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + if (cfield != TS_NULL_MLOC) {
> + compression_acceptable = 0;
> + nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
> + for (i=0; i<nvalues; i++) {
> + value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len);
> + if (!value) {
> + continue;
> + }
> +
> + if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
> + compression_acceptable = 1;
> + *compress_type = COMPRESSION_TYPE_DEFLATE;
> + break;
> + } else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) {
> + compression_acceptable = 1;
> + *compress_type = COMPRESSION_TYPE_GZIP;
> + break;
> + }
> + }
> +
> + TSHandleMLocRelease(cbuf, chdr, cfield);
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> +
> + if (!compression_acceptable) {
> + info("no acceptable encoding found in request header, not compressible");
> + return 0;
> + }
> + } else {
> + info("no acceptable encoding found in request header, not compressible");
> + TSHandleMLocRelease(cbuf, chdr, cfield);
> + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
> + return 0;
> + }
> +
> + if (server) {
> + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
> + } else {
> + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
> + }
> +
> + /* If there already exists a content encoding then we don't want
> + to do anything. */
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
> + if (field_loc) {
> + info("response is already content encoded, not compressible");
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return 0;
> + }
> +
> + /* We only want to do gzip compression on documents that have a
> + content type of "text/" or "application/x-javascript". */
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
> + if (!field_loc) {
> + info("no content type header found, not compressible");
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return 0;
> + }
> +
> + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len);
> +
> + int rv = host_configuration->ContentTypeIsCompressible(value, len);
> + if (!rv) {
> + info("content-type [%.*s] not compressible", len, value);
> + }
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + return rv;
> +}
> +
> +
> +static void
> +gzip_transform_add(TSHttpTxn txnp, int /* server ATS_UNUSED */, HostConfiguration * hc, int compress_type)
> +{
> + int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked);
> + if (tmp) {
> + //happens on cache_stale_hit
> + debug("transform hook already set, bail");
> + return;
> + } else {
> + TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &GZIP_ONE);
> + info("adding compression transform");
> + }
> +
> + TSHttpTxnUntransformedRespCache(txnp, 1);
> +
> + if (!hc->cache()) {
> + debug("@@@@@ Gzip cache disabled");
> + TSHttpTxnTransformedRespCache(txnp, 0);
> + } else {
> + TSMBuffer bufp;
> + TSMLoc hdr_loc, field_loc;
> + int cache = 1;
> + if (TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS || TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) {
> + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "@gzip_nocache", strlen("@gzip_nocache"));
> + if (field_loc) {
> + cache = 0;
> + debug("@@@@@ Gzip disallows cacheing of transformed response");
> + TSHandleMLocRelease(bufp, hdr_loc, field_loc);
> + } else {
> + debug("@@@@ Gzip allows cacheing of transformed response");
> + }
> + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
> + }
> +
> + TSHttpTxnTransformedRespCache(txnp, cache);
> + }
> +
> + TSVConn connp;
> + GzipData *data;
> +
> + connp = TSTransformCreate(gzip_transform, txnp);
> + data = gzip_data_alloc(compress_type);
> + data->txn = txnp;
> +
> + TSContDataSet(connp, data);
> + TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
> +}
> +
> +static int
> +cache_transformable(TSHttpTxn txnp)
> +{
> + int obj_status;
> +
> + if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
> + warning("Couldn't get cache status of object");
> + return 0;
> + }
> + if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) {
> + info("stale cache hit");
> + return 0;
> + }
> + if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
> + info("fresh cache hit");
> + return 1;
> + }
> +
> + return 0;
> +}
> +
> +HostConfiguration *
> +find_host_configuration(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer bufp, TSMLoc locp)
> +{
> + TSMLoc fieldp = TSMimeHdrFieldFind(bufp, locp, TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST);
> +
> + if (fieldp) {
> + int strl;
> + const char *strv = TSMimeHdrFieldValueStringGet(bufp, locp, fieldp, -1, &strl);
> + TSHandleMLocRelease(bufp, locp, fieldp);
> +
> + HostConfiguration * host_configuration = config->Find(strv, strl);
> + return host_configuration;
> + }
> +
> + return config->GlobalConfiguration();
> +}
> +
> +
> +static int
> +transform_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata)
> +{
> + TSHttpTxn txnp = (TSHttpTxn) edata;
> + int compress_type = COMPRESSION_TYPE_DEFLATE;
> +
> + switch (event) {
> + case TS_EVENT_HTTP_READ_REQUEST_HDR:
> + {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + int url_len;
> + char * url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
> + HostConfiguration * hc = find_host_configuration(txnp, req_buf, req_loc);
> + //we could clone the hosting configuration here, to make it deletable on reload?
> + TSHttpTxnArgSet(txnp, arg_idx_host_configuration, (void *) hc);
> +
> + if (!hc->enabled() || !hc->IsUrlAllowed(url, url_len)) {
> + //FIXME: no double negatives
> + TSHttpTxnArgSet(txnp, arg_idx_url_disallowed, (void *) &GZIP_ONE);
> + info("url [%.*s] not allowed", url_len, url);
> + } else {
> + normalize_accept_encoding(txnp, req_buf, req_loc);
> + }
> + TSfree(url);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_READ_RESPONSE_HDR:
> + {
> + //os: the accept encoding header needs to be restored..
> + //otherwise the next request won't get a cache hit on this
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if (hc != NULL) {
> + if (hc->remove_accept_encoding()) {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + restore_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + }
> +
> + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
> + if ( allowed && gzip_transformable(txnp, 1, hc, &compress_type)) {
> + gzip_transform_add(txnp, 1, hc, compress_type);
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_SEND_REQUEST_HDR:
> + {
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if (hc!=NULL) {
> + if (hc->remove_accept_encoding()) {
> + TSMBuffer req_buf;
> + TSMLoc req_loc;
> + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
> + hide_accept_encoding(txnp, req_buf, req_loc, global_hidden_header_name);
> + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
> + }
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
> + {
> + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed);
> + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, arg_idx_host_configuration);
> + if ( hc != NULL ) {
> + if (allowed && cache_transformable(txnp) && gzip_transformable(txnp, 0, hc, &compress_type)) {
> + gzip_transform_add(txnp, 0, hc, compress_type);
> + }
> + }
> + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
> + }
> + break;
> +
> + default:
> + fatal("gzip transform unknown event");
> + }
> +
> + return 0;
> +}
> +
> +
> +static void
> +read_configuration(TSCont contp) {
> + const char * path = (const char *)TSContDataGet(contp);
> + Configuration * newconfig = Configuration::Parse(path);
> +
> + Configuration * oldconfig =__sync_lock_test_and_set(&config, newconfig);
> + debug("config swapped,old config %p", oldconfig);
> +
> + //FIXME: we have leaked.
> + //consider cloning or refcounting the configuration passed to the txn
> + //to make deleting the old configuration possible
> + //if (config != NULL )
> + // delete config;
> +}
> +
> +static int
> +management_update(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
> +{
> + TSReleaseAssert(event == TS_EVENT_MGMT_UPDATE);
> + info("management update event received");
> + read_configuration(contp);
> + return 0;
> +}
> +
> +
> +void
> +TSPluginInit(int argc, const char *argv[])
> +{
> + string config_path;
> +
> + if (argc > 2) {
> + fatal("the gzip plugin does not accept more than 1 plugin argument");
> + } else if (argc == 2) {
> + config_path = std::string(argv[1]);
> + }
> +
> + info("TSPluginInit %s", argv[0]);
> +
> + if (!register_plugin()) {
> + fatal("The gzip plugin failed to register");
> + }
> +
> + //if (argc == 2) {
> + // dictionary = load_dictionary(argv[1]);
> + //}
> +
> + if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", &arg_idx_hooked) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> + if (TSHttpArgIndexReserve("gzip", "for storing if compression is applicable", &arg_idx_host_configuration) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> + if (TSHttpArgIndexReserve("gzip", "for storing if compression is disallowed for this txn", &arg_idx_url_disallowed) != TS_SUCCESS) {
> + fatal("failed to reserve an argument index");
> + }
> +
> + global_hidden_header_name = init_hidden_header_name();
> +
> + TSCont management_contp = TSContCreate(management_update, NULL);
> + //fixme: never freed. there is no shutdown event?
> + char * p = (char*)TSmalloc(config_path.size()+1);
> + strcpy(p,config_path.c_str());
> + TSContDataSet(management_contp,(void*)p);
> + TSMgmtUpdateRegister(management_contp, TAG);
> + read_configuration(management_contp);
> +
> + TSCont transform_contp = TSContCreate(transform_plugin, NULL);
> + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp);
> + TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp);
> +
> + info("loaded");
> +}
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.config
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/gzip.config b/plugins/experimental/ats_speed/gzip/gzip.config
> new file mode 100644
> index 0000000..81e9fa4
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/gzip.config
> @@ -0,0 +1,6 @@
> +# Set some global options first
> +cache true
> +enabled true
> +remove-accept-encoding false
> +compressible-content-type text/*
> +compressible-content-type *javascript*
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.cc
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/misc.cc b/plugins/experimental/ats_speed/gzip/misc.cc
> new file mode 100644
> index 0000000..0ea6911
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/misc.cc
> @@ -0,0 +1,197 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#include "ts/ts.h"
> +//#include "ink_defs.h"
> +#include <stdint.h>
> +#include <inttypes.h>
> +#include "misc.h"
> +#include <string.h>
> +#include "debug_macros.h"
> +
> +voidpf
> +gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size)
> +{
> + return (voidpf) TSmalloc(items * size);
> +}
> +
> +void
> +gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address)
> +{
> + TSfree(address);
> +}
> +
> +void
> +normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + int deflate = 0;
> + int gzip = 0;
> +
> + //remove the accept encoding field(s),
> + //while finding out if gzip or deflate is supported.
> + while (field) {
> + TSMLoc tmp;
> +
> + if (!deflate && !gzip) {
> + int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field);
> +
> + while (value_count > 0) {
> + int val_len = 0;
> + const char *val;
> +
> + --value_count;
> + val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
> +
> + if (val_len == (int) strlen("gzip"))
> + gzip = !strncmp(val, "gzip", val_len);
> + else if (val_len == (int) strlen("deflate"))
> + deflate = !strncmp(val, "deflate", val_len);
> + }
> + }
> +
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +
> + //append a new accept-encoding field in the header
> + if (deflate || gzip) {
> + TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> +
> + if (gzip) {
> + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
> + info("normalized accept encoding to gzip");
> + } else if (deflate) {
> + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", strlen("deflate"));
> + info("normalized accept encoding to deflate");
> + }
> +
> + TSMimeHdrFieldAppend(reqp, hdr_loc, field);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + }
> +}
> +
> +void
> +hide_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + while (field) {
> + TSMLoc tmp;
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +}
> +
> +void
> +restore_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name)
> +{
> + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
> +
> + while (field) {
> + TSMLoc tmp;
> + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
> + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
> + TSHandleMLocRelease(reqp, hdr_loc, field);
> + field = tmp;
> + }
> +}
> +
> +const char *
> +init_hidden_header_name()
> +{
> + char * hidden_header_name;
> + const char *var_name = "proxy.config.proxy_name";
> + TSMgmtString result;
> +
> + if (TSMgmtStringGet(var_name, &result) != TS_SUCCESS) {
> + fatal("failed to get server name");
> + } else {
> + int hidden_header_name_len = strlen("x-accept-encoding-") + strlen(result);
> + hidden_header_name = (char *) TSmalloc(hidden_header_name_len + 1);
> + hidden_header_name[hidden_header_name_len] = 0;
> + sprintf(hidden_header_name, "x-accept-encoding-%s", result);
> + }
> + return hidden_header_name;
> +}
> +
> +int
> +register_plugin()
> +{
> + TSPluginRegistrationInfo info;
> +
> + info.plugin_name = (char*)"gzip";
> + info.vendor_name = (char*)"Apache";
> + info.support_email = (char*)"dev@trafficserver.apache.org";
> +
> + if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
> + return 0;
> + }
> + return 1;
> +}
> +
> +const char *
> +load_dictionary(const char *preload_file)
> +{
> + char *dict = (char *) malloc(800000);
> + uLong dictId = adler32(0L, Z_NULL, 0);
> + uLong *adler = &dictId;
> +
> + FILE *fp;
> + int i = 0;
> +
> + fp = fopen(preload_file, "r");
> + if (!fp) {
> + fatal("gzip-transform: ERROR: Unable to open dict file %s", preload_file);
> + }
> +
> + /* dict = (char *) calloc(8000, sizeof(char)); */
> +
> + i = 0;
> + while (!feof(fp)) {
> + if (fscanf(fp, "%s\n", dict + i) == 1) {
> + i = strlen(dict);
> + strcat(dict + i, " ");
> + ++i;
> + }
> + }
> + dict[i - 1] = '\0';
> +
> + /* TODO get the adler compute right */
> + *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
> + return dict;
> +}
> +
> +void
> +gzip_log_ratio(int64_t in, int64_t out)
> +{
> + // if (in) {
> + // info("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, ((float) (in - out) / in));
> + //} else {
> + // debug("Compressed size %PRId64 (bytes), Original size %" PRId64", ratio: %f", out, in, 0.0F);
> + // }
> +}
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.h
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/gzip/misc.h b/plugins/experimental/ats_speed/gzip/misc.h
> new file mode 100644
> index 0000000..c44fb6b
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/gzip/misc.h
> @@ -0,0 +1,84 @@
> +/** @file
> +
> + Transforms content using gzip or deflate
> +
> + @section license License
> +
> + Licensed to the Apache Software Foundation (ASF) under one
> + or more contributor license agreements. See the NOTICE file
> + distributed with this work for additional information
> + regarding copyright ownership. The ASF licenses this file
> + to you under the Apache License, Version 2.0 (the
> + "License"); you may not use this file except in compliance
> + with the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> + */
> +
> +#ifndef _GZIP_MISC_H_
> +#define _GZIP_MISC_H_
> +
> +#define __STDC_LIMIT_MACROS
> +#define __STDC_CONSTANT_MACROS
> +#include <stdint.h>
> +#include <zlib.h>
> +#include <ts/ts.h>
> +#include <stdlib.h> //exit()
> +#include <stdio.h>
> +
> +//zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html
> +static const int ZLIB_MEMLEVEL = 9; //min=1 (optimize for memory),max=9 (optimized for speed)
> +static const int WINDOW_BITS_DEFLATE = -15;
> +static const int WINDOW_BITS_GZIP = 31;
> +
> +//misc
> +static const int COMPRESSION_TYPE_DEFLATE = 1;
> +static const int COMPRESSION_TYPE_GZIP = 2;
> +//this one is just for txnargset/get to point to
> +static const int GZIP_ONE = 1;
> +static const int DICT_PATH_MAX = 512;
> +static const int DICT_ENTRY_MAX = 2048;
> +
> +//this one is used to rename the accept encoding header
> +//it will be restored later on
> +//to make it work, the name must be different then downstream proxies though
> +//otherwise the downstream will restore the accept encoding header
> +
> +enum transform_state
> +{
> + transform_state_initialized,
> + transform_state_output,
> + transform_state_finished
> +};
> +
> +typedef struct
> +{
> + TSHttpTxn txn;
> + TSVIO downstream_vio;
> + TSIOBuffer downstream_buffer;
> + TSIOBufferReader downstream_reader;
> + int downstream_length;
> + z_stream zstrm;
> + enum transform_state state;
> + int compression_type;
> +} GzipData;
> +
> +
> +voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
> +void gzip_free(voidpf opaque, voidpf address);
> +void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
> +void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
> +void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name);
> +const char * init_hidden_header_name();
> +int check_ts_version();
> +int register_plugin();
> +const char *load_dictionary(const char *preload_file);
> +void gzip_log_ratio(int64_t in, int64_t out);
> +
> +#endif
>
> http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> ----------------------------------------------------------------------
> diff --git a/plugins/experimental/ats_speed/scripts/prepare_psol.sh b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> new file mode 100755
> index 0000000..4862249
> --- /dev/null
> +++ b/plugins/experimental/ats_speed/scripts/prepare_psol.sh
> @@ -0,0 +1,95 @@
> +#!/bin/bash
> +#
> +# Copyright 2012 Google Inc.
> +#
> +# Licensed under the Apache License, Version 2.0 (the "License");
> +# you may not use this file except in compliance with the License.
> +# You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +#
> +# Author: jefftk@google.com (Jeff Kaufman)
> +# Author: oschaaf@we-amp.com (Otto van der Schaaf)
> +#
> +# Usage:
> +# scripts/prepare_psol.sh /path/to/mod_pagespeed/src
> +#
> +# Creates a directory psol/ and copies headers and a few source files from a
> +# depot_tools (glient) checkout into psol/include. Along with creating
> +# binaries, this is a step in preparing psol.tar.gz for distribution.
> +#
> +
> +set -u # check for undefined variables
> +set -e # exit on failed commands
> +
> +if [ "$(basename "$PWD")" != "ats_speed" ] ; then
> + echo "$(basename $0) must be invoked from the ats_speed directory"
> + exit 1
> +fi
> +
> +if [ $# -ne 1 ] ; then
> + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
> + exit 1
> +fi
> +
> +MOD_PAGESPEED_SRC="$1"
> +
> +if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
> + basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
> + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
> + exit 1
> +fi
> +
> +if [ -e psol ] ; then
> + echo "A psol/ directory already exists. Move it somewhere else and rerun."
> + exit 1
> +fi
> +mkdir psol/
> +# Copy over the .h files, plus a few selected .cc and .c files.
> +rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
> + --exclude=".svn" \
> + --exclude=".git" \
> + --include='*.h' \
> + --include='*/' \
> + --include="apr_thread_compatible_pool.cc" \
> + --include="serf_url_async_fetcher.cc" \
> + --include="apr_mem_cache.cc" \
> + --include="key_value_codec.cc" \
> + --include="apr_memcache2.c" \
> + --include="loopback_route_fetcher.cc" \
> + --include="add_headers_fetcher.cc" \
> + --include="console_css_out.cc" \
> + --include="console_out.cc" \
> + --include="dense_hash_map" \
> + --include="dense_hash_set" \
> + --include="sparse_hash_map" \
> + --include="sparse_hash_set" \
> + --include="sparsetable" \
> + --include="mod_pagespeed_console_out.cc" \
> + --include="mod_pagespeed_console_css_out.cc" \
> + --include="mod_pagespeed_console_html_out.cc" \
> + --exclude='*'
> +mkdir -p psol/lib/Debug/linux/ia32
> +mkdir -p psol/lib/Debug/linux/x64
> +mkdir -p psol/lib/Release/linux/ia32
> +mkdir -p psol/lib/Release/linux/x64
> +
> +# Log that we did this.
> +SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
> +SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL | awk -F/ '{print $(NF-1)}')"
> +
> +DATE="$(date +%F)"
> +echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
> + >> psol/include_history.txt
> +
> +echo
> +echo "Output is in psol/include. Now put binaries in psol/lib following"
> +echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
> +echo "and then you can distribute PSOL."
> +
>