You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by os...@apache.org on 2012/09/01 15:03:12 UTC
git commit: TS-1431 the gzip plugin needs cleanup and refactoring
Updated Branches:
refs/heads/master 3c35457c2 -> d59c7b9b1
TS-1431 the gzip plugin needs cleanup and refactoring
Tested: Ubuntu 12.04
Autor: Otto van der Schaaf <os...@apache.org>
- added debug macros
- moved some stuff from gzip.c into misc.h/misc.cc for clarity
- renamed some stuff for clarity
- fixed a valgrind error on initialising a txnarg
- sanitized plugin diagnostic output and code comments
- attempt to conform the code more to the guidelines
- named some magick numbers
- removed unused crc field
- disabled custom dictionaries, as that code seems not ready for production.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/d59c7b9b
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/d59c7b9b
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/d59c7b9b
Branch: refs/heads/master
Commit: d59c7b9b17251eae4a4a29020c079875a9da0c61
Parents: 3c35457
Author: Otto van der Schaaf <os...@apache.org>
Authored: Sat Sep 1 14:55:30 2012 +0200
Committer: Otto van der Schaaf <os...@apache.org>
Committed: Sat Sep 1 14:55:30 2012 +0200
----------------------------------------------------------------------
plugins/experimental/gzip/debug_macros.h | 32 +
plugins/experimental/gzip/gzip.c | 845 +++++++++----------------
plugins/experimental/gzip/misc.c | 184 ++++++
plugins/experimental/gzip/misc.h | 60 ++
4 files changed, 577 insertions(+), 544 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d59c7b9b/plugins/experimental/gzip/debug_macros.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/gzip/debug_macros.h b/plugins/experimental/gzip/debug_macros.h
new file mode 100644
index 0000000..4efd961
--- /dev/null
+++ b/plugins/experimental/gzip/debug_macros.h
@@ -0,0 +1,32 @@
+#ifndef _DBG_MACROS_H
+#define _DBG_MACROS_H
+
+#include <ts/ts.h>
+
+#define TAG "gzip"
+
+#define debug(fmt, args...) do { \
+ TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ } while (0)
+
+#define info(fmt, args...) do { \
+ TSDebug(TAG, "INFO: " fmt, ##args ); \
+ } while (0)
+
+#define warning(fmt, args...) do { \
+ TSDebug(TAG, "WARNING: " fmt, ##args ); \
+} while (0)
+
+#define error(fmt, args...) do { \
+ TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+} while (0)
+
+#define fatal(fmt, args...) do { \
+ TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args ); \
+ exit(-1); \
+} while (0)
+
+
+#endif //_DBG_MACROS_H
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d59c7b9b/plugins/experimental/gzip/gzip.c
----------------------------------------------------------------------
diff --git a/plugins/experimental/gzip/gzip.c b/plugins/experimental/gzip/gzip.c
index d80d722..6071f68 100644
--- a/plugins/experimental/gzip/gzip.c
+++ b/plugins/experimental/gzip/gzip.c
@@ -32,95 +32,30 @@
#include <zlib.h>
#include <ts/ts.h>
#include <inttypes.h>
-
-#define DICT_PATH_MAX 512
-#define DICT_ENTRY_MAX 2048
-
-#define COMPRESSION_TYPE_DEFLATE 1
-#define COMPRESSION_TYPE_GZIP 2
-
-#define MOD_GZIP_ZLIB_CFACTOR 9
-#define MOD_GZIP_ZLIB_BSIZE 8096
-
-/* ZLIB's deflate() compression algorithm uses the same */
-/* 0-9 based scale that GZIP does where '1' is 'Best speed' */
-/* and '9' is 'Best compression'. Testing has proved level '6' */
-/* to be about the best level to use in an HTTP Server. */
-
-
-//os: FIXME: look into autoscaling the compression level based on connection speed
-//a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
-//might be served better with little or no compression at all
-#define MOD_GZIP_DEFLATE_DEFAULT_COMPRESSION_LEVEL 6
-#define CACHE_TRANSFORMED_RESPONSES 0
-#define REMOVE_SERVER_REQUEST_ENCODING 1
-
-const char* PLUGIN_NAME = "gzip";
+#include "debug_macros.h"
+#include "misc.h"
+
+//FIXME: custom dictionaries would be nice to have.
+//currently, custom dictionaries are completely disabled
+//FIXME: look into autoscaling the compression level based on connection speed
+// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
+// might be served better with little or no compression at all
+//FIXME: look into compressing from the task thread pool
+//FIXME: make configurion for which mime types are compressible
+//FIXME: make removing the accept encoding configurable
+//FIXME: make caching of compressed responses configurable
+
+// from mod_deflate:
+// ZLIB's compression algorithm uses a
+// 0-9 based scale that GZIP does where '1' is 'Best speed'
+// and '9' is 'Best compression'. Testing has proved level '6'
+// to be about the best level to use in an HTTP Server.
+const int ZLIB_COMPRESSION_LEVEL = 6;
+const int CACHE_TRANSFORMED_RESPONSES = 0;
+const int REMOVE_SERVER_REQUEST_ENCODING = 1;
int arg_idx_hooked;
-int hook_set = 1;
-char * hidden_header_name;
-
-typedef struct
-{
- TSHttpTxn txn;
- TSVIO output_vio;
- TSIOBuffer output_buffer;
- TSIOBufferReader output_reader;
- int output_length;
- z_stream zstrm;
- uLong crc; //os: unused, we are using raw deflate
- int state;
- int compression_type;
-} GzipData;
-
-
-char preload_file[1024];
-uLong dictId;
-int preload = 0;
-char dictionary[800000];
-
-void
-load_dictionary(char *dict, uLong *adler)
-{
- FILE *fp;
- int i = 0;
-
- fp = fopen(preload_file, "r");
- if (!fp) {
- TSError("gzip-transform: ERROR: Unable to open dict file %s\n", preload_file);
- exit(0);
- }
-
- /* dict = (char *) calloc(8000, sizeof(char)); */
-
- i = 0;
- while (!feof(fp)) {
- if (fscanf(fp, "%s\n", dict + i) == 1) {
- i = strlen(dict);
- strcat(dict + i, " ");
- ++i;
- }
- }
- dict[i - 1] = '\0';
-
- /* TODO get the adler compute right */
- *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
-}
-
-static voidpf
-gzip_alloc(voidpf opaque, uInt items, uInt size)
-{
- return (voidpf) TSmalloc(items * size);
-}
-
-
-static void
-gzip_free(voidpf opaque, voidpf address)
-{
- TSfree(address);
-}
-
+const char *dictionary = NULL;
static GzipData *
gzip_data_alloc(int compression_type)
@@ -128,16 +63,13 @@ gzip_data_alloc(int compression_type)
GzipData *data;
int err;
- TSDebug(PLUGIN_NAME, "gzip_data_alloc() start");
-
- data = (GzipData *)TSmalloc(sizeof(GzipData));
- data->output_vio = NULL;
- data->output_buffer = NULL;
- data->output_reader = NULL;
- data->output_length = 0;
- data->state = 0;
+ data = (GzipData *) TSmalloc(sizeof(GzipData));
+ data->downstream_vio = NULL;
+ data->downstream_buffer = NULL;
+ data->downstream_reader = NULL;
+ data->downstream_length = 0;
+ data->state = transform_state_initialized;
data->compression_type = compression_type;
- data->crc = crc32(0L, Z_NULL, 0); //os: not used?
data->zstrm.next_in = Z_NULL;
data->zstrm.avail_in = 0;
data->zstrm.total_in = 0;
@@ -149,31 +81,18 @@ gzip_data_alloc(int compression_type)
data->zstrm.opaque = (voidpf) 0;
data->zstrm.data_type = Z_ASCII;
- int window_size = -15;//deflate
- if (compression_type == COMPRESSION_TYPE_GZIP)
- window_size = 15 + 16;//gzip
-
- TSDebug("gzip","initializing window size %d", window_size);
+ int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
- err = deflateInit2(
- &data->zstrm,
- MOD_GZIP_DEFLATE_DEFAULT_COMPRESSION_LEVEL,
- Z_DEFLATED,
- window_size,
- MOD_GZIP_ZLIB_CFACTOR,
- Z_DEFAULT_STRATEGY);
+ err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
if (err != Z_OK) {
- TSDebug("gzip", "deflate init error %d", err);
- TSError("gzip-transform: ERROR: deflateInit (%d)!", err);
- exit(1);
+ fatal("gzip-transform: ERROR: deflateInit (%d)!", err);
}
- if (preload) {
- TSAssert(&data->zstrm);
+ if (dictionary) {
err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
if (err != Z_OK) {
- TSError("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
+ fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
}
}
@@ -182,86 +101,74 @@ gzip_data_alloc(int compression_type)
static void
-gzip_data_destroy(GzipData *data)
+gzip_data_destroy(GzipData * data)
{
- int err;
+ TSReleaseAssert(data);
- if (data) {
- err = deflateEnd(&data->zstrm);
+ //deflateEnd returnvalue ignore is intentional
+ //it would spew log on every client abort
+ deflateEnd(&data->zstrm);
- //os: this can happen when clients abort.
- // that is not very uncommon, so don't log it.
- if (err != Z_OK) {
- //TSDebug(PLUGIN_NAME,"gzip-transform: ERROR: deflateEnd (%d)!", err);
- //TSError("gzip-transform: ERROR: deflateEnd (%d)!", err);
- }
-
- if (data->output_buffer)
- TSIOBufferDestroy(data->output_buffer);
- TSfree(data);
+ if (data->downstream_buffer) {
+ TSIOBufferDestroy(data->downstream_buffer);
}
+
+ TSfree(data);
}
+//FIXME: ensure the headers are set/appended correctly
+//FIXME: some things are potentially compressible. those responses
+//FIXME: the etag alteration isn't proper. it should modify the value inside quotes
+// specify a very header..
static void
-gzip_transform_init(TSCont contp, GzipData *data)
+gzip_transform_init(TSCont contp, GzipData * data)
{
- TSVConn output_conn;
+ //update the vary, content-encoding, and etag response headers
+ //prepare the downstream for transforming
+
+ TSVConn downstream_conn;
TSMBuffer bufp;
TSMLoc hdr_loc;
- TSMLoc ce_loc; /* for the content encoding mime field */
+ TSMLoc ce_loc; /* for the content encoding mime field */
- TSDebug(PLUGIN_NAME, "gzip_transform_init");
+ data->state = transform_state_output;
- data->state = 1;
-
- /* Mark the output data as having deflate content encoding */
TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc);
-
- //FIXME: these todo's
- //os: this is a little rough around the edges.
- //this should insert/append field values as needed instead.
-
- //Probably should check for errors
- TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
+ //FIXME: Probably should check for errors
+ TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Content-Encoding", -1);
if (data->compression_type == COMPRESSION_TYPE_DEFLATE) {
TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", -1);
} else if (data->compression_type == COMPRESSION_TYPE_GZIP) {
TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "gzip", -1);
- }
+ }
TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
-
- //os: error checking. formally -> this header should be send for any document,
- //that will potentially alternate on compression?
-
- TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
+ TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Vary", -1);
TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", -1);
TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
-
- //os: since we alter the entity body, update the etag to something different as well
ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
if (ce_loc) {
int changetag = 1;
int strl;
- const char * strv = TSMimeHdrFieldValueStringGet(bufp,hdr_loc,ce_loc,-1,&strl);
+ const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl);
//do not alter weak etags.
//FIXME: consider just making the etag weak for compressed content
- if (strl>=2) {
- if ( ( strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/' ) {
- changetag=0;
+ if (strl >= 2) {
+ if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') {
+ changetag = 0;
}
if (changetag) {
- TSMimeHdrFieldValueAppend(bufp,hdr_loc,ce_loc,0,"-df",3);
+ TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3);
}
TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
@@ -269,141 +176,130 @@ gzip_transform_init(TSCont contp, GzipData *data)
}
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
-
- /* Get the output connection where we'll write data to. */
- output_conn = TSTransformOutputVConnGet(contp);
- data->output_buffer = TSIOBufferCreate();
- data->output_reader = TSIOBufferReaderAlloc(data->output_buffer);
- data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader, INT64_MAX);
+ downstream_conn = TSTransformOutputVConnGet(contp);
+ data->downstream_buffer = TSIOBufferCreate();
+ data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
+ data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX);
}
+
static void
-gzip_transform_one(GzipData *data, TSIOBufferReader input_reader, int amount)
+gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int amount)
{
- TSIOBufferBlock blkp;
- const char *ibuf;
- char *obuf;
- int64_t ilength, olength;
+ TSIOBufferBlock downstream_blkp;
+ const char *upstream_buffer;
+ char *downstream_buffer;
+ int64_t upstream_length, downstream_length;
int err;
while (amount > 0) {
- blkp = TSIOBufferReaderStart(input_reader);
-
- /* TSIOBufferReaderStart may return an error pointer */
- if (!blkp) {
- TSDebug(PLUGIN_NAME, "couldn't get from IOBufferBlock");
- TSError("couldn't get from IOBufferBlock");
- return;
+ downstream_blkp = TSIOBufferReaderStart(upstream_reader);
+ if (!downstream_blkp) {
+ error("couldn't get from IOBufferBlock");
+ return;
}
- ibuf = TSIOBufferBlockReadStart(blkp, input_reader, &ilength);
-
- /* TSIOBufferReaderStart may return an error pointer */
- if (!ibuf) {
- TSDebug(PLUGIN_NAME, "couldn't get from TSIOBufferBlockReadStart");
- TSError("couldn't get from TSIOBufferBlockReadStart");
- return;
+ upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
+ if (!upstream_buffer) {
+ error("couldn't get from TSIOBufferBlockReadStart");
+ return;
}
-
- if (ilength > amount) {
- ilength = amount;
+
+ if (upstream_length > amount) {
+ upstream_length = amount;
}
- data->zstrm.next_in = (unsigned char *) ibuf;
- data->zstrm.avail_in = ilength;
+ data->zstrm.next_in = (unsigned char *) upstream_buffer;
+ data->zstrm.avail_in = upstream_length;
while (data->zstrm.avail_in > 0) {
- blkp = TSIOBufferStart(data->output_buffer);
-
- obuf = TSIOBufferBlockWriteStart(blkp, &olength);
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
- data->zstrm.next_out = (unsigned char *) obuf;
- data->zstrm.avail_out = olength;
+ data->zstrm.next_out = (unsigned char *) downstream_buffer;
+ data->zstrm.avail_out = downstream_length;
- /* Encode */
err = deflate(&data->zstrm, Z_NO_FLUSH);
-
+
if (err != Z_OK)
- TSDebug(PLUGIN_NAME,"deflate() call failed: %d", err);
-
- if (olength > data->zstrm.avail_out) {
- TSIOBufferProduce(data->output_buffer, olength - data->zstrm.avail_out);
- data->output_length += (olength - data->zstrm.avail_out);
+ warning("deflate() call failed: %d", err);
+
+ if (downstream_length > data->zstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
+ data->downstream_length += (downstream_length - data->zstrm.avail_out);
}
-
+
if (data->zstrm.avail_out > 0) {
if (data->zstrm.avail_in != 0) {
- TSError("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
+ error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
}
}
}
- /* compute CRC for error checking at client */
- data->crc = crc32(data->crc, (unsigned char *) ibuf, ilength);
-
- TSIOBufferReaderConsume(input_reader, ilength);
- amount -= ilength;
+ TSIOBufferReaderConsume(upstream_reader, upstream_length);
+ amount -= upstream_length;
}
}
-
static void
-gzip_transform_finish(GzipData *data)
+gzip_transform_finish(GzipData * data)
{
- TSDebug(PLUGIN_NAME, "gzip_transform_finish");
- if (data->state == 1) {
- TSIOBufferBlock blkp;
- char *obuf;
- int64_t olength;
+ if (data->state == transform_state_output) {
+ TSIOBufferBlock downstream_blkp;
+ char *downstream_buffer;
+ int64_t downstream_length;
int err;
- data->state = 2;
+ data->state = transform_state_finished;
for (;;) {
- blkp = TSIOBufferStart(data->output_buffer);
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
- obuf = TSIOBufferBlockWriteStart(blkp, &olength);
- data->zstrm.next_out = (unsigned char *) obuf;
- data->zstrm.avail_out = olength;
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+ data->zstrm.next_out = (unsigned char *) downstream_buffer;
+ data->zstrm.avail_out = downstream_length;
- /* Encode remaining data */
err = deflate(&data->zstrm, Z_FINISH);
- if (olength > data->zstrm.avail_out) {
- TSIOBufferProduce(data->output_buffer, olength - data->zstrm.avail_out);
- data->output_length += (olength - data->zstrm.avail_out);
+ if (downstream_length > data->zstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
+ data->downstream_length += (downstream_length - data->zstrm.avail_out);
}
if (err == Z_OK) { /* some more data to encode */
continue;
}
- /* done! */
+
if (err != Z_STREAM_END) {
- TSDebug(PLUGIN_NAME, "deflate should report Z_STREAM_END");
+ warning("deflate should report Z_STREAM_END");
}
break;
}
- if (data->output_length != (data->zstrm.total_out)) {
- TSError("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->output_length,
- data->zstrm.total_out);
+ if (data->downstream_length != (data->zstrm.total_out)) {
+ error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
+ data->zstrm.total_out);
}
- if (data->compression_type == COMPRESSION_TYPE_GZIP) {
- char*p = (char *)&(data->zstrm.adler);
+
+ if (data->compression_type == COMPRESSION_TYPE_GZIP) {
+ info("append gzip adler");
+ char *p = (char *) &(data->zstrm.adler);
int length = 8;
while (length > 0) {
- obuf = TSIOBufferBlockWriteStart (blkp, &olength);
- if (olength > length)
- olength = length;
- memcpy (obuf, p, olength);
- p += olength;
- length -= olength;
- TSIOBufferProduce (data->output_buffer, olength);
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+ if (downstream_length > length)
+ downstream_length = length;
+ memcpy(downstream_buffer, p, downstream_length);
+ p += downstream_length;
+ length -= downstream_length;
+ TSIOBufferProduce(data->downstream_buffer, downstream_length);
}
- data->output_length += 8;
+ data->downstream_length += 8;
}
+
+ gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
}
}
@@ -411,66 +307,62 @@ gzip_transform_finish(GzipData *data)
static void
gzip_transform_do(TSCont contp)
{
- TSVIO write_vio;
+ TSVIO upstream_vio;
GzipData *data;
- int64_t towrite;
- int64_t avail;
- int64_t length;
-
+ int64_t upstream_todo;
+ int64_t upstream_avail;
+ int64_t downstream_bytes_written;
+
data = TSContDataGet(contp);
- if (data->state == 0) {
+ if (data->state == transform_state_initialized) {
gzip_transform_init(contp, data);
}
- write_vio = TSVConnWriteVIOGet(contp);
+ upstream_vio = TSVConnWriteVIOGet(contp);
+ downstream_bytes_written = data->downstream_length;
- length = data->output_length;
-
- if (!TSVIOBufferGet(write_vio)) {
+ if (!TSVIOBufferGet(upstream_vio)) {
gzip_transform_finish(data);
- TSVIONBytesSet(data->output_vio, data->output_length);
- TSDebug(PLUGIN_NAME, "Compressed size %d (bytes)", data->output_length);
+ TSVIONBytesSet(data->downstream_vio, data->downstream_length);
- if (data->output_length > length) {
- TSVIOReenable(data->output_vio);
+ if (data->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(data->downstream_vio);
}
return;
}
- towrite = TSVIONTodoGet(write_vio);
+ upstream_todo = TSVIONTodoGet(upstream_vio);
- if (towrite > 0) {
- avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio));
+ if (upstream_todo > 0) {
+ upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
- if (towrite > avail) {
- towrite = avail;
+ if (upstream_todo > upstream_avail) {
+ upstream_todo = upstream_avail;
}
- if (towrite > 0) {
- gzip_transform_one(data, TSVIOReaderGet(write_vio), towrite);
- TSVIONDoneSet(write_vio, TSVIONDoneGet(write_vio) + towrite);
+ if (upstream_todo > 0) {
+ gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
+ TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
}
}
- if (TSVIONTodoGet(write_vio) > 0) {
-
- if (towrite > 0) {
- if (data->output_length > length) {
- TSVIOReenable(data->output_vio);
- }
- TSContCall(TSVIOContGet(write_vio), TS_EVENT_VCONN_WRITE_READY, write_vio);
+ if (TSVIONTodoGet(upstream_vio) > 0) {
+ if (upstream_todo > 0) {
+ if (data->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(data->downstream_vio);
+ }
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
}
} else {
gzip_transform_finish(data);
- TSVIONBytesSet(data->output_vio, data->output_length);
- TSDebug(PLUGIN_NAME, "gzip_transform_do-> Compressed size %d (bytes)", data->output_length);
+ TSVIONBytesSet(data->downstream_vio, data->downstream_length);
- if (data->output_length > length) {
- TSVIOReenable(data->output_vio);
+ if (data->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(data->downstream_vio);
}
- TSContCall(TSVIOContGet(write_vio), TS_EVENT_VCONN_WRITE_COMPLETE, write_vio);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
}
}
@@ -484,28 +376,24 @@ gzip_transform(TSCont contp, TSEvent event, void *edata)
return 0;
} else {
switch (event) {
- case TS_EVENT_ERROR:
- {
- TSDebug(PLUGIN_NAME, "gzip_transform: TS_EVENT_ERROR starts");
- TSVIO write_vio = TSVConnWriteVIOGet(contp);
- TSContCall(TSVIOContGet(write_vio), TS_EVENT_ERROR, write_vio);
+ case TS_EVENT_ERROR:{
+ debug("gzip_transform: TS_EVENT_ERROR starts");
+ TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
}
break;
-
case TS_EVENT_VCONN_WRITE_COMPLETE:
TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
break;
-
case TS_EVENT_VCONN_WRITE_READY:
- gzip_transform_do(contp);
- break;
-
+ gzip_transform_do(contp);
+ break;
case TS_EVENT_IMMEDIATE:
- gzip_transform_do(contp);
- break;
-
+ gzip_transform_do(contp);
+ break;
default:
- gzip_transform_do(contp);
+ warning("unknown event [%d]", event);
+ gzip_transform_do(contp);
break;
}
}
@@ -517,50 +405,6 @@ gzip_transform(TSCont contp, TSEvent event, void *edata)
static int
gzip_transformable(TSHttpTxn txnp, int server)
{
- // ToDo: This is pretty ugly, do we need a new scope here? XXX
- {
- TSMBuffer bufp;
- TSMLoc hdr_loc;
- TSHttpStatus resp_status;
- TSMLoc con_field;
- int con_len;
- const char *con_val;
-
- if (server) {
- TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
- } else {
- TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
- }
- resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
-
- con_field = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONNECTION, -1);
- if (con_field) {
- con_val = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, con_field, 0, &con_len);
- TSHandleMLocRelease(bufp, hdr_loc, con_field);
-
- //OS: !!!!!! FIXME !!!!!!!
- //this is a hotfix for some weird behavior from an origin
- //needs to be patched properly. this disables support for transactions that send the connection:close header
- if (con_val && con_len == 5)
- {
- TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
- return -999;
- }
- }
-
- if (TS_HTTP_STATUS_OK == resp_status) {
- if (TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc) != TS_SUCCESS) {
- TSError("Unable to release handle to server request");
- }
- //return 1;
- } else {
- if (TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc) != TS_SUCCESS) {
- TSError("Unable to release handle to server request");
- }
- return -100;
- }
- }
-
/* Server response header */
TSMBuffer bufp;
TSMLoc hdr_loc;
@@ -573,47 +417,71 @@ gzip_transformable(TSHttpTxn txnp, int server)
const char *value;
int nvalues;
- int i, deflate_flag, len;
+ int i, compression_acceptable, len;
+ TSHttpStatus resp_status;
+ if (server) {
+ TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
+ } else {
+ TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
+ }
+ resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
+
+ //conservatively pick some statusses to compress
+ if (!(resp_status == 200 || resp_status == 404 || resp_status == 500)) {
+ info("http response status [%d] is not compressible", resp_status);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return 0;
+ }
+
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
- /* check if client accepts "deflate" */
+ //the only compressible method is currently GET.
+ int method_length;
+ const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length);
+ if (method != TS_HTTP_METHOD_GET) {
+ debug("method is not GET, not compressible");
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+ return 0;
+ }
+
cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
- if (TS_NULL_MLOC != cfield) {
+ if (cfield != TS_NULL_MLOC) {
nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
-
+
value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, 0, &len);
- deflate_flag = 0;
+ compression_acceptable = 0;
i = 0;
while (nvalues > 0) {
if (value && (strncasecmp(value, "deflate", strlen("deflate")) == 0)) {
- deflate_flag = 1;
+ compression_acceptable = 1;
+ break;
+ } else if (value && (strncasecmp(value, "gzip", strlen("gzip")) == 0)) {
+ compression_acceptable = 1;
break;
- } else if (value && (strncasecmp(value, "gzip", strlen("gzip")) == 0)){
- deflate_flag = 1;
- break;
}
++i;
value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len);
-
--nvalues;
}
- if (!deflate_flag) {
+ if (!compression_acceptable) {
+ info("no acceptable encoding found in request header, not compressible");
TSHandleMLocRelease(cbuf, chdr, cfield);
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
- return -7;
+ return 0;
}
TSHandleMLocRelease(cbuf, chdr, cfield);
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
} else {
-
+ info("no acceptable encoding found in request header, not compressible");
TSHandleMLocRelease(cbuf, chdr, cfield);
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
- return -6;
+ return 0;
}
if (server) {
@@ -626,11 +494,12 @@ gzip_transformable(TSHttpTxn txnp, int server)
to do anything. */
field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
if (field_loc) {
+ info("response is already content encoded, not compressible");
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
- return -3;
+ return 0;
}
-
+
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
/* We only want to do gzip compression on documents that have a
@@ -638,28 +507,29 @@ gzip_transformable(TSHttpTxn txnp, int server)
field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
if (!field_loc) {
+ info("no content type header found, not compressible");
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
- return -4;
+ return 0;
}
value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len);
/*os: FIXME -> vary:accept-encoding needs to be added if any of these contenttypes is encountered
- */
- if ( len >= 5 && value && (strncasecmp(value, "text/", sizeof("text/") - 1) == 0)) {
+ */
+ if (len >= 5 && (strncasecmp(value, "text/", sizeof("text/") - 1) == 0)) {
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
- TSDebug(PLUGIN_NAME, "@ transformable, it is text/*");
- return 0;
- } else if ( len >= (sizeof("application/x-javascript") -1) && value && (strncasecmp(value, "application/x-javascript", (sizeof("application/x-javascript") - 1)) == 0)) {
+ info("response is text/*, will compress");
+ return 1;
+ } else if (strncasecmp(value, "application/x-javascript", (sizeof("application/x-javascript") - 1)) == 0) {
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
- TSDebug(PLUGIN_NAME, "@ transformable, it is application/x-javascript");
- return 0;
+ info("response is application/x-javascript, will compress");
+ return 1;
} else {
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
- TSDebug(PLUGIN_NAME, "@ not a transformable content type");
- return -5;
+ info("content type [%.*s] => will not compress", len, value);
+ return 0;
}
}
@@ -667,176 +537,85 @@ gzip_transformable(TSHttpTxn txnp, int server)
static void
gzip_transform_add(TSHttpTxn txnp, int server)
{
- int * tmp = (int *)TSHttpTxnArgGet(txnp, arg_idx_hooked);
+ int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked);
+ if (tmp) {
+ //happens on cache_stale_hit
+ debug("transform hook already set, bail");
+ return;
+ } else {
+ TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &HOOK_SET);
+ info("adding compression transform");
+ }
- if ( CACHE_TRANSFORMED_RESPONSES ) {
+ if (CACHE_TRANSFORMED_RESPONSES) {
TSHttpTxnUntransformedRespCache(txnp, 1);
TSHttpTxnTransformedRespCache(txnp, 0);
- } else {
+ } else {
TSHttpTxnTransformedRespCache(txnp, 0);
TSHttpTxnUntransformedRespCache(txnp, 1);
}
- if ( tmp ) {
- TSDebug("gzip", "hook already set, bail");
- return;
- } else {
- TSHttpTxnArgSet(txnp, arg_idx_hooked, &hook_set);
- TSDebug("gzip", "adding compression transform");
- }
-
TSVConn connp;
GzipData *data;
- TSDebug(PLUGIN_NAME,"zip_transform_add -> tstransformcreate()");
connp = TSTransformCreate(gzip_transform, txnp);
-
- TSDebug(PLUGIN_NAME,"zip_transform_add -> gzip_data_alloc()");
-
/* Client request header */
TSMBuffer cbuf;
TSMLoc chdr;
TSMLoc cfield;
int len;
- int gzip=0;
+ int gzip = 0;
TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, 0, &len);
- if (len == 4) gzip=1;
- TSHandleMLocRelease(cbuf, chdr, cfield);
- TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
-
+ //we normalized to either deflate or gzip. only gzip has length 4
+ if (len == strlen("gzip"))
+ gzip = 1;
+ TSHandleMLocRelease(cbuf, chdr, cfield);
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
data = gzip_data_alloc(gzip ? COMPRESSION_TYPE_GZIP : COMPRESSION_TYPE_DEFLATE);
data->txn = txnp;
-
- TSDebug(PLUGIN_NAME,"zip_transform_add -> TSContDataSet()");
- TSContDataSet(connp, data);
- TSDebug(PLUGIN_NAME,"zip_transform_add -> TSHttpTxnHookAdd()");
+ TSContDataSet(connp, data);
TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
}
-static void
-normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
-{
- TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
- int deflate = 0;
- int gzip = 0;
-
- //remove the accept encoding field(s),
- //while finding out if deflate is supported.
- while (field) {
- TSMLoc tmp;
-
- if (!deflate && !gzip) {
- int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field);
-
- while (value_count > 0) {
- int val_len = 0;
- const char* val;
-
- --value_count;
- val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
-
- if (val_len == strlen("gzip"))
- gzip = !strncmp(val, "gzip", val_len);
- else if (val_len == strlen("deflate"))
- deflate = !strncmp(val, "deflate", val_len);
- }
- }
-
- tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
- TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
- TSHandleMLocRelease(reqp, hdr_loc, field);
- field = tmp;
- }
-
- //append a new accept-encoding field in the header
- if (deflate || gzip){
- TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
- TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
-
- if (gzip) {
- TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
- TSDebug("gzip","normalized accept encoding to gzip");
- }
- else if (deflate) {
- TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", strlen("deflate"));
- TSDebug("gzip","normalized accept encoding to deflate");
- }
-
- TSMimeHdrFieldAppend(reqp, hdr_loc, field);
- TSHandleMLocRelease(reqp, hdr_loc, field);
- }
-}
-
static int
-cache_transformable(TSHttpTxn txnp) {
+cache_transformable(TSHttpTxn txnp)
+{
int obj_status;
if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
- TSError("[%s] Couldn't get cache status of object", __FUNCTION__);
- TSDebug("gzip_lu","[%s] Couldn't get cache status of object", __FUNCTION__);
+ warning("Couldn't get cache status of object");
return 0;
}
- if ((obj_status == TS_CACHE_LOOKUP_HIT_FRESH) /*|| (obj_status == TS_CACHE_LOOKUP_HIT_STALE)*/) {
- TSDebug("gzip_lu", "[%s] doc found in cache, will add transformation", __FUNCTION__);
- return 1;
+ if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) {
+ info("stale cache hit");
+ return 0;
}
- TSDebug("gzip_lu", "[%s] cache object's status is %d; not transformable",
- __FUNCTION__, obj_status);
- return 0;
-}
-
-static void
-kill_request_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
-{
- TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING
- , TS_MIME_LEN_ACCEPT_ENCODING);
-
- while (field) {
- TSMLoc tmp;
- tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
-
-
- TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
- TSHandleMLocRelease(reqp, hdr_loc, field);
- field = tmp;
+ if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
+ info("fresh cache hit");
+ return 1;
}
-}
-
-static void
-restore_request_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
-{
- TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
-
- while (field) {
- TSMLoc tmp;
- tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
-
- TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
- TSHandleMLocRelease(reqp, hdr_loc, field);
- field = tmp;
- }
+ debug("unhandled cache status, returning untransformable %d", obj_status);
+ return 0;
}
static int
transform_plugin(TSCont contp, TSEvent event, void *edata)
{
TSHttpTxn txnp = (TSHttpTxn) edata;
- int reason;
-
+
switch (event) {
- case TS_EVENT_HTTP_READ_REQUEST_HDR:
- {
+ case TS_EVENT_HTTP_READ_REQUEST_HDR:{
TSMBuffer req_buf;
TSMLoc req_loc;
- if ( TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS ){
+ if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
normalize_accept_encoding(txnp, req_buf, req_loc);
TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
}
@@ -844,61 +623,49 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
}
break;
- case TS_EVENT_HTTP_READ_RESPONSE_HDR: {
- //os: boy oh boy. the accept encoding header needs to be restored
- //otherwise alt selection will fail. hopefully a better solution exists
- //then this header shuffle
- TSMBuffer req_buf;
- TSMLoc req_loc;
-
- if ( TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS ){
- if (REMOVE_SERVER_REQUEST_ENCODING) {
- restore_request_accept_encoding(txnp, req_buf, req_loc);
+ case TS_EVENT_HTTP_READ_RESPONSE_HDR:{
+ //os: the accept encoding header needs to be restored..
+ //otherwise the next request won't get a cache hit on this
+ TSMBuffer req_buf;
+ TSMLoc req_loc;
+
+ if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
+ if (REMOVE_SERVER_REQUEST_ENCODING) {
+ restore_accept_encoding(txnp, req_buf, req_loc);
+ }
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
}
- TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
- }
- reason = gzip_transformable(txnp, 1);
- if (reason >= 0) {
- TSDebug(PLUGIN_NAME, "server content transformable");
- gzip_transform_add(txnp, 1);
- } else {
- TSDebug(PLUGIN_NAME, "server content NOT transformable [%d]", reason);
+ if (gzip_transformable(txnp, 1)) {
+ gzip_transform_add(txnp, 1);
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
}
+ break;
- TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
- } break;
-
- case TS_EVENT_HTTP_SEND_REQUEST_HDR: {
- if (REMOVE_SERVER_REQUEST_ENCODING) {
- TSMBuffer req_buf;
- TSMLoc req_loc;
- if ( TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS ){
- kill_request_accept_encoding(txnp, req_buf, req_loc);
- TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ case TS_EVENT_HTTP_SEND_REQUEST_HDR:{
+ if (REMOVE_SERVER_REQUEST_ENCODING) {
+ TSMBuffer req_buf;
+ TSMLoc req_loc;
+ if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
+ hide_accept_encoding(txnp, req_buf, req_loc);
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ }
}
- }
- TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
- } break;
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ }
+ break;
case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
- reason = cache_transformable(txnp);
- if (reason) {
- reason = gzip_transformable(txnp, 0);
- if (reason >= 0) {
- TSDebug("gzip-transform", "cached content transformable");
- gzip_transform_add(txnp, 0);
- } else {
- TSDebug("gzip-transform", "cached data: forwarding unchanged (%d)", reason);
- }
+ if (cache_transformable(txnp) && gzip_transformable(txnp, 0)) {
+ gzip_transform_add(txnp, 0);
}
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
break;
default:
- TSError("gzip transform unknown event, exit!");
- exit(1);
+ fatal("gzip transform unknown event");
}
return 0;
@@ -908,43 +675,33 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
void
TSPluginInit(int argc, const char *argv[])
{
- dictId = adler32(0L, Z_NULL, 0);
- if (argc == 2) {
- strcpy(preload_file, argv[1]);
- preload = 1;
- load_dictionary(dictionary, &dictId);
- }
-
- TSDebug(PLUGIN_NAME, "gzip plugin loads");
-
+ if (argc > 1)
+ fatal("the gzip plugin does not accept plugin arguments");
- TSDebug("gzip-transform", "gzip plugin loads");
+ info("TSPluginInit %s", argv[0]);
- if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", &arg_idx_hooked) != TS_SUCCESS) {
- TSError("failed to reserve an argument index");
- exit(-1);
+ if (!register_plugin()) {
+ fatal("The gzip plugin failed to register");
+ }
+ if (!check_ts_version()) {
+ fatal("The gzip plugin requires at least traffic server v3");
}
- const char* var_name = "proxy.config.proxy_name";
- TSMgmtString result;
-
- if ( TSMgmtStringGet( var_name, &result) != TS_SUCCESS ) {
- TSDebug("gzip", "failed to get server name");
- exit(-1);
- } else {
- TSDebug("gzip", "got server name: %s", result);
+ if (argc == 2) {
+ dictionary = load_dictionary(argv[1]);
+ }
- int hidden_header_name_len = strlen("x-accept-encoding-") + strlen(result);
- hidden_header_name = (char *)TSmalloc( hidden_header_name_len + 1 );
- result[hidden_header_name_len] = 0;
- sprintf( hidden_header_name, "x-accept-encoding-%s", result);
- TSDebug("gzip", "hidden header name: %s / %ld", hidden_header_name, strlen(hidden_header_name));
+ if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", &arg_idx_hooked) != TS_SUCCESS) {
+ fatal("failed to reserve an argument index");
}
+ init_hidden_header_name();
TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(transform_plugin, NULL));
TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(transform_plugin, NULL));
TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(transform_plugin, NULL));
TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, TSContCreate(transform_plugin, NULL));
+
+ info("loaded");
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d59c7b9b/plugins/experimental/gzip/misc.c
----------------------------------------------------------------------
diff --git a/plugins/experimental/gzip/misc.c b/plugins/experimental/gzip/misc.c
new file mode 100644
index 0000000..de2bb76
--- /dev/null
+++ b/plugins/experimental/gzip/misc.c
@@ -0,0 +1,184 @@
+#include "misc.h"
+#include <string.h>
+#include <ts/ts.h>
+#include "debug_macros.h"
+
+voidpf
+gzip_alloc(voidpf opaque, uInt items, uInt size)
+{
+ return (voidpf) TSmalloc(items * size);
+}
+
+void
+gzip_free(voidpf opaque, voidpf address)
+{
+ TSfree(address);
+}
+
+void
+normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ int deflate = 0;
+ int gzip = 0;
+
+ //remove the accept encoding field(s),
+ //while finding out if gzip or deflate is supported.
+ while (field) {
+ TSMLoc tmp;
+
+ if (!deflate && !gzip) {
+ int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field);
+
+ while (value_count > 0) {
+ int val_len = 0;
+ const char *val;
+
+ --value_count;
+ val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
+
+ if (val_len == strlen("gzip"))
+ gzip = !strncmp(val, "gzip", val_len);
+ else if (val_len == strlen("deflate"))
+ deflate = !strncmp(val, "deflate", val_len);
+ }
+ }
+
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+
+ //append a new accept-encoding field in the header
+ if (deflate || gzip) {
+ TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+
+ if (gzip) {
+ TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
+ info("normalized accept encoding to gzip");
+ } else if (deflate) {
+ TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", strlen("deflate"));
+ info("normalized accept encoding to deflate");
+ }
+
+ TSMimeHdrFieldAppend(reqp, hdr_loc, field);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ }
+}
+
+void
+hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ while (field) {
+ TSMLoc tmp;
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+}
+
+void
+restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
+{
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1);
+
+ while (field) {
+ TSMLoc tmp;
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+}
+
+void
+init_hidden_header_name()
+{
+ const char *var_name = "proxy.config.proxy_name";
+ TSMgmtString result;
+
+ if (TSMgmtStringGet(var_name, &result) != TS_SUCCESS) {
+ fatal("failed to get server name");
+ } else {
+ int hidden_header_name_len = strlen("x-accept-encoding-") + strlen(result);
+ hidden_header_name = (char *) TSmalloc(hidden_header_name_len + 1);
+ hidden_header_name[hidden_header_name_len] = 0;
+ sprintf(hidden_header_name, "x-accept-encoding-%s", result);
+ }
+}
+
+int
+check_ts_version()
+{
+ const char *ts_version = TSTrafficServerVersionGet();
+ TSReleaseAssert(ts_version);
+
+ int scan_result;
+ int major_version;
+
+ scan_result = sscanf(ts_version, "%d", &major_version);
+ TSReleaseAssert(scan_result == 1);
+
+ return major_version >= 3;
+}
+
+int
+register_plugin()
+{
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = "gzip";
+ info.vendor_name = "Apache";
+ info.support_email = "dev@trafficserver.apache.org";
+
+ if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
+ return 0;
+ }
+ return 1;
+}
+
+const char *
+load_dictionary(const char *preload_file)
+{
+ char *dict = (char *) malloc(800000);
+ uLong dictId = adler32(0L, Z_NULL, 0);
+ uLong *adler = &dictId;
+
+ FILE *fp;
+ int i = 0;
+
+ fp = fopen(preload_file, "r");
+ if (!fp) {
+ fatal("gzip-transform: ERROR: Unable to open dict file %s", preload_file);
+ }
+
+ /* dict = (char *) calloc(8000, sizeof(char)); */
+
+ i = 0;
+ while (!feof(fp)) {
+ if (fscanf(fp, "%s\n", dict + i) == 1) {
+ i = strlen(dict);
+ strcat(dict + i, " ");
+ ++i;
+ }
+ }
+ dict[i - 1] = '\0';
+
+ /* TODO get the adler compute right */
+ *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
+ return dict;
+}
+
+void
+gzip_log_ratio(int64_t in, int64_t out)
+{
+ if (in) {
+ info("Compressed size %ld (bytes), Original size %ld, ratio: %f", out, in, ((float) (in - out) / in));
+ } else {
+ debug("Compressed size %ld (bytes), Original size %ld, ratio: %f", out, in, 0.0F);
+ }
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d59c7b9b/plugins/experimental/gzip/misc.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/gzip/misc.h b/plugins/experimental/gzip/misc.h
new file mode 100644
index 0000000..89421f2
--- /dev/null
+++ b/plugins/experimental/gzip/misc.h
@@ -0,0 +1,60 @@
+#ifndef _GZIP_MISC_H_
+#define _GZIP_MISC_H_
+
+#include <zlib.h>
+#include <ts/ts.h>
+#include <stdlib.h> //exit()
+#include <stdio.h>
+
+
+//zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html
+static const int ZLIB_MEMLEVEL = 9; //min=1 (optimize for memory),max=9 (optimized for speed)
+static const int WINDOW_BITS_DEFLATE = -15;
+static const int WINDOW_BITS_GZIP = 31;
+
+//misc
+static const int COMPRESSION_TYPE_DEFLATE = 1;
+static const int COMPRESSION_TYPE_GZIP = 2;
+static const int HOOK_SET = 1;
+static const int DICT_PATH_MAX = 512;
+static const int DICT_ENTRY_MAX = 2048;
+
+//this one is used to rename the accept encoding header
+//it will be restored later on
+//to make it work, the name must be different then downstream proxies though
+//otherwise the downstream will restore the accept encoding header
+char *hidden_header_name;
+
+
+enum transform_state
+{
+ transform_state_initialized,
+ transform_state_output,
+ transform_state_finished
+};
+
+typedef struct
+{
+ TSHttpTxn txn;
+ TSVIO downstream_vio;
+ TSIOBuffer downstream_buffer;
+ TSIOBufferReader downstream_reader;
+ int downstream_length;
+ z_stream zstrm;
+ enum transform_state state;
+ int compression_type;
+} GzipData;
+
+
+voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
+void gzip_free(voidpf opaque, voidpf address);
+void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
+void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
+void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc);
+void init_hidden_header_name();
+int check_ts_version();
+int register_plugin();
+const char *load_dictionary(const char *preload_file);
+void gzip_log_ratio(int64_t in, int64_t out);
+
+#endif