You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by ig...@apache.org on 2012/03/07 16:08:00 UTC
git commit: TS-1129 gzip (or deflate) plugin Author: Otto van der
Schaaf
Updated Branches:
refs/heads/master 77b7d3505 -> d0315d104
TS-1129 gzip (or deflate) plugin
Author: Otto van der Schaaf
Adding Otto's new and improved version of the gzip plugin.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/commit/d0315d10
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/tree/d0315d10
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/diff/d0315d10
Branch: refs/heads/master
Commit: d0315d104e9f9e240f320542a19ddaeace68d088
Parents: 77b7d35
Author: Igor Galić <i....@brainsware.org>
Authored: Wed Mar 7 15:12:05 2012 +0000
Committer: Igor Galić <i....@brainsware.org>
Committed: Wed Mar 7 15:12:05 2012 +0000
----------------------------------------------------------------------
Makefile | 2 +-
gzip/LICENSE | 202 ++++++++++++
gzip/Makefile | 26 ++
gzip/README | 14 +
gzip/gzip.c | 911 ++++++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 1154 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/blob/d0315d10/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index 386e6d3..17d0878 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ TAR = tar chof - "$$tardir"
SUBDIRS = balancer buffer_upload cacheurl combo_handler esi geoip_acl \
header_filter hipes mysql_remap regex_remap \
stale_while_revalidate stats_over_http \
- memcached_remap
+ memcached_remap gzip
all:
@for plugin in $(SUBDIRS); do \
http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/blob/d0315d10/gzip/LICENSE
----------------------------------------------------------------------
diff --git a/gzip/LICENSE b/gzip/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/gzip/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/blob/d0315d10/gzip/Makefile
----------------------------------------------------------------------
diff --git a/gzip/Makefile b/gzip/Makefile
new file mode 100644
index 0000000..2696871
--- /dev/null
+++ b/gzip/Makefile
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TSXS?=tsxs
+
+all: gzip.c
+ $(TSXS) -v -c $? -o gzip.so
+
+install:
+ $(TSXS) -i -o gzip.so
+
+clean:
+ rm -f *.lo *.so
http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/blob/d0315d10/gzip/README
----------------------------------------------------------------------
diff --git a/gzip/README b/gzip/README
new file mode 100644
index 0000000..ea30188
--- /dev/null
+++ b/gzip/README
@@ -0,0 +1,14 @@
+What this plugin does
+=====================
+
+in spite of its name, it compresses responses to the raw deflate format.
+it also normalizes the accept-encoding header from user agent requests
+to either deflate or nothing. it compresses responses from origins only,
+though it can be modified easily to also compress contents from cache.
+
+Major issues
+============
+- there is an issue with *some* origins that send connection:close.
+ the plugin will not execute in that situation as a temporary fix.
+- the workings of this plugin support our use case, but probably is not
+ on size fits all. it shouldnt be too hard to adjust it to your own needs though.
http://git-wip-us.apache.org/repos/asf/trafficserver-plugins/blob/d0315d10/gzip/gzip.c
----------------------------------------------------------------------
diff --git a/gzip/gzip.c b/gzip/gzip.c
new file mode 100644
index 0000000..b9e95c0
--- /dev/null
+++ b/gzip/gzip.c
@@ -0,0 +1,911 @@
+/** @file
+
+ Transforms content using gzip
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+
+// WHAT THIS PLUGIN DOES:
+// in spite of its name, it compresses responses to the raw deflate format.
+// it also normalizes the accept-encoding header from user agent requests
+// to either deflate or nothing. it compresses responses from origins only,
+// though it can be modified easily to also compress contents from cache.
+//
+// MAJOR ISSUES:
+// - there is an issue with *some* origins that send connection:close.
+// the plugin will not execute in that situation as a temporary fix.
+// - the workings of this plugin support our use case, but probably is not
+// on size fits all. it shouldnt be too hard to adjust it to your own needs though.
+
+
+
+
+#include <limits.h>
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/file.h>
+#include <sys/types.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include <assert.h>
+#include <ts/ts.h>
+
+#define DICT_PATH_MAX 512
+#define DICT_ENTRY_MAX 2048
+
+typedef struct
+{
+ TSHttpTxn txn;
+ TSVIO output_vio;
+ TSIOBuffer output_buffer;
+ TSIOBufferReader output_reader;
+ int output_length;
+ z_stream zstrm;
+ uLong crc; //os: unused, we are using raw deflate
+ int state;
+} GzipData;
+
+
+char preload_file[1024];
+uLong dictId;
+int preload = 0;
+char dictionary[800000];
+
+void
+load_dictionary(char *dict, uLong * adler)
+{
+ FILE *fp;
+ int i = 0;
+
+ fp = fopen(preload_file, "r");
+ if (!fp) {
+ TSError("gzip-transform: ERROR: Unable to open dict file %s\n", preload_file);
+ exit(0);
+ }
+
+ /* dict = (char *) calloc(8000,sizeof(char)); */
+
+ i = 0;
+ while (!feof(fp)) {
+ if (fscanf(fp, "%s\n", dict + i) == 1) {
+ i = strlen(dict);
+ strcat(dict + i, " ");
+ i++;
+ }
+ }
+ dict[i - 1] = '\0';
+
+ /* TODO get the adler compute right */
+ *adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
+}
+
+static voidpf
+gzip_alloc(voidpf opaque, uInt items, uInt size)
+{
+ TSDebug("gzip", "gzip_alloc()");
+ return (voidpf) TSmalloc(items * size);
+}
+
+
+static void
+gzip_free(voidpf opaque, voidpf address)
+{
+ TSDebug("gzip", "gzip_free() start");
+ TSfree(address);
+}
+
+
+static GzipData *
+gzip_data_alloc()
+{
+ TSDebug("gzip", "gzip_data_alloc() start");
+ GzipData *data;
+ int err;
+
+ data = (GzipData *) TSmalloc(sizeof(GzipData));
+ data->output_vio = NULL;
+ data->output_buffer = NULL;
+ data->output_reader = NULL;
+ data->output_length = 0;
+ data->state = 0;
+ data->crc = crc32(0L, Z_NULL, 0);
+
+ data->zstrm.next_in = Z_NULL;
+ data->zstrm.avail_in = 0;
+ data->zstrm.total_in = 0;
+ data->zstrm.next_out = Z_NULL;
+ data->zstrm.avail_out = 0;
+ data->zstrm.total_out = 0;
+ data->zstrm.zalloc = gzip_alloc;
+ data->zstrm.zfree = gzip_free;
+ data->zstrm.opaque = (voidpf) 0;
+ data->zstrm.data_type = Z_ASCII;
+
+#define MOD_GZIP_ZLIB_WINDOWSIZE -15
+#define MOD_GZIP_ZLIB_CFACTOR 9
+#define MOD_GZIP_ZLIB_BSIZE 8096
+
+/* ZLIB's deflate() compression algorithm uses the same */
+/* 0-9 based scale that GZIP does where '1' is 'Best speed' */
+/* and '9' is 'Best compression'. Testing has proved level '6' */
+/* to be about the best level to use in an HTTP Server. */
+
+
+//os: FIXME: look into autoscaling the compression level based on connection speed
+//a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
+//might be served better with little or no compression at all
+#define MOD_GZIP_DEFLATE_DEFAULT_COMPRESSION_LEVEL 6
+ err = deflateInit2(
+ &data->zstrm,
+ MOD_GZIP_DEFLATE_DEFAULT_COMPRESSION_LEVEL,
+ Z_DEFLATED,
+ MOD_GZIP_ZLIB_WINDOWSIZE,
+ MOD_GZIP_ZLIB_CFACTOR,
+ Z_DEFAULT_STRATEGY);
+
+ if (err != Z_OK) {
+ TSError("gzip-transform: ERROR: deflateInit (%d)!", err);
+ exit(1);
+ }
+
+ if (preload) {
+ assert(&data->zstrm);
+ err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
+ if (err != Z_OK) {
+ TSError("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
+ }
+ }
+ TSDebug("gzip", "gzip_data_alloc() end");
+ return data;
+}
+
+
+static void
+gzip_data_destroy(GzipData * data)
+{
+ TSDebug("gzip","gzip-transform: gzip_data_destroy() starts");
+ int err;
+
+ if (data) {
+ err = deflateEnd(&data->zstrm);
+
+ //os: this can happen when clients abort.
+ // that is not very uncommon, so don't log it.
+ if (err != Z_OK) {
+ //TSDebug("gzip","gzip-transform: ERROR: deflateEnd (%d)!", err);
+ //TSError("gzip-transform: ERROR: deflateEnd (%d)!", err);
+ }
+
+ if (data->output_buffer)
+ TSIOBufferDestroy(data->output_buffer);
+ TSfree(data);
+ }
+
+ TSDebug("gzip","gzip-transform: gzip_data_destroy() ends");
+}
+
+
+static void
+gzip_transform_init(TSCont contp, GzipData * data)
+{
+ TSDebug("gzip", "gzip_transform_init\n");
+ TSVConn output_conn;
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+ TSMLoc ce_loc; /* for the content encoding mime field */
+
+ data->state = 1;
+
+ /* Mark the output data as having deflate content encoding */
+
+ TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc);
+
+
+ //FIXME: these todo's
+ //os: this is a little rough around the edges.
+ //this should insert/append field values as needed instead.
+
+ //Probably should check for errors
+ TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
+ TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Content-Encoding", -1);
+ TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", -1);
+ TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+
+
+ //os: error checking. formally -> this header should be send for any document,
+ //that will potentially alternate on compression?
+
+ TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc);
+ TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Vary", -1);
+ TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "Accept-Encoding", -1);
+ TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+
+
+ //os: since we alter the entity body, update the etag to something different as well
+ ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
+ if (ce_loc){
+ TSMimeHdrFieldValueAppend(bufp,hdr_loc,ce_loc,0,"-df",3);
+ TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
+ }
+
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+
+ /* Get the output connection where we'll write data to. */
+ output_conn = TSTransformOutputVConnGet(contp);
+
+ data->output_buffer = TSIOBufferCreate();
+ data->output_reader = TSIOBufferReaderAlloc(data->output_buffer);
+ data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader, INT64_MAX);
+}
+
+
+static void
+gzip_transform_one(GzipData * data, TSIOBufferReader input_reader, int amount)
+{
+ TSDebug("gzip", "gzip_transform_one\n");
+ TSIOBufferBlock blkp;
+ const char *ibuf;
+ char *obuf;
+ int64_t ilength, olength;
+ int err;
+
+ while (amount > 0) {
+ TSDebug("gzip", "gzip_transform_one->loop, amount: %d\n",amount);
+ blkp = TSIOBufferReaderStart(input_reader);
+
+ /* TSIOBufferReaderStart may return an error pointer */
+ if (!blkp) {
+ TSDebug("gzip", "couldn't get from IOBufferBlock");
+ TSError("couldn't get from IOBufferBlock\n");
+ return;
+ }
+
+ TSDebug("gzip", "gzip_transform_one->TSIOBufferReaderStart finished");
+ ibuf = TSIOBufferBlockReadStart(blkp, input_reader, &ilength);
+
+ /* TSIOBufferReaderStart may return an error pointer */
+ if (!ibuf) {
+ TSDebug("gzip", "couldn't get from TSIOBufferBlockReadStart");
+ TSError("couldn't get from TSIOBufferBlockReadStart\n");
+ return;
+ }
+
+
+ TSDebug("gzip", "gzip_transform_one->TSIOBufferBlockReadStart ilength: %d\n",ilength);
+
+ if (ilength > amount) {
+ ilength = amount;
+ }
+
+ data->zstrm.next_in = (unsigned char *) ibuf;
+ data->zstrm.avail_in = ilength;
+
+ while (data->zstrm.avail_in > 0) {
+ TSDebug("gzip", "gzip_transform_one->Tdata->zstrm.avail_in: %d\n",data->zstrm.avail_in);
+ blkp = TSIOBufferStart(data->output_buffer);
+
+ obuf = TSIOBufferBlockWriteStart(blkp, &olength);
+
+ data->zstrm.next_out = (unsigned char *) obuf;
+ data->zstrm.avail_out = olength;
+
+ /* Encode */
+ err = deflate(&data->zstrm, Z_NO_FLUSH);
+
+ if (err != Z_OK)
+ TSDebug("gzip","deflate() call failed: %d",err);
+
+ if (olength > data->zstrm.avail_out) {
+ TSIOBufferProduce(data->output_buffer, olength - data->zstrm.avail_out);
+ data->output_length += (olength - data->zstrm.avail_out);
+ }
+
+ TSDebug("gzip","deflate() call values olength: %d, ilength: %d, data->output_length: %d",olength, ilength, data->output_length );
+
+ if (data->zstrm.avail_out > 0) {
+ if (data->zstrm.avail_in != 0) {
+ TSError("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
+ }
+ }
+ }
+ TSDebug("gzip", "gzip_transform_one pre compute crc");
+ /* compute CRC for error checking at client */
+ data->crc = crc32(data->crc, (unsigned char *) ibuf, ilength);
+
+ TSDebug("gzip", "gzip_transform_one pre consume %d from reader",ilength);
+ TSIOBufferReaderConsume(input_reader, ilength);
+ amount -= ilength;
+ }
+ TSDebug("gzip", "gzip_transform_one survived");
+}
+
+
+static void
+gzip_transform_finish(GzipData * data)
+{
+ TSDebug("gzip", "gzip_transform_finish\n");
+ if (data->state == 1) {
+ TSIOBufferBlock blkp;
+ char *obuf;
+ int64_t olength;
+ int err;
+
+ data->state = 2;
+
+ for (;;) {
+ blkp = TSIOBufferStart(data->output_buffer);
+
+ obuf = TSIOBufferBlockWriteStart(blkp, &olength);
+ data->zstrm.next_out = (unsigned char *) obuf;
+ data->zstrm.avail_out = olength;
+
+ /* Encode remaining data */
+ err = deflate(&data->zstrm, Z_FINISH);
+
+ if (olength > data->zstrm.avail_out) {
+ TSIOBufferProduce(data->output_buffer, olength - data->zstrm.avail_out);
+ data->output_length += (olength - data->zstrm.avail_out);
+ }
+
+ if (err == Z_OK) { /* some more data to encode */
+ continue;
+ }
+ /* done! */
+ if (err != Z_STREAM_END) {
+ TSDebug("gzip-transform", "deflate should report Z_STREAM_END\n");
+ }
+ break;
+ }
+
+ if (data->output_length != (data->zstrm.total_out)) {
+ TSError("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->output_length,
+ data->zstrm.total_out);
+ }
+
+ /* compute/append crc to end of stream */
+
+ /*
+ blkp = TSIOBufferStart (data->output_buffer);
+ char crcbuf[8];
+ char* buf=crcbuf;
+ uLong tmp = data->crc;
+ buf[0] = tmp & 0xff; tmp >>= 8;
+ buf[1] = tmp & 0xff; tmp >>= 8;
+ buf[2] = tmp & 0xff; tmp >>= 8;
+ buf[3] = tmp & 0xff;
+
+ tmp = data->zstrm.total_in;
+ buf[4] = tmp & 0xff; tmp >>= 8;
+ buf[5] = tmp & 0xff; tmp >>= 8;
+ buf[6] = tmp & 0xff; tmp >>= 8;
+ buf[7] = tmp & 0xff;
+
+ char*p = buf;
+ int length;
+ length = 8;
+
+ while (length > 0) {
+ obuf = TSIOBufferBlockWriteStart (blkp, &olength);
+ if (olength > length) {
+ olength = length;
+ }
+
+ memcpy (obuf, p, olength);
+ p += olength;
+ length -= olength;
+
+ TSIOBufferProduce (data->output_buffer, olength);
+ }
+
+ data->output_length += 8;*/
+
+ }
+}
+
+
+static void
+gzip_transform_do(TSCont contp)
+{
+ TSDebug("gzip", "gzip_transform_do\n");
+ TSVIO write_vio;
+ GzipData *data;
+ int towrite;
+ int avail;
+ int length;
+
+ /* Get our data structure for this operation. The private data
+ structure contains the output vio and output buffer. If the
+ private data structure pointer is NULL, then we'll create it
+ and initialize its internals. */
+ data = TSContDataGet(contp);
+ if (data->state == 0) {
+ TSDebug("gzip-transform", "gzip_transform_do: data->state == 0, get_transform_init");
+ gzip_transform_init(contp, data);
+ } else
+ {
+ TSDebug("gzip-transform", "gzip_transform_do: data->state == %d, NO get_transform_init", data->state);
+ }
+
+ /* Get the write vio for the write operation that was performed on
+ ourself. This vio contains the buffer that we are to read from
+ as well as the continuation we are to call when the buffer is
+ empty. */
+ write_vio = TSVConnWriteVIOGet(contp);
+
+ length = data->output_length;
+ TSDebug("gzip-transform", "gzip_transform_do: first length: %d",length);
+ /* We also check to see if the write vio's buffer is non-NULL. A
+ NULL buffer indicates that the write operation has been
+ shutdown and that the continuation does not want us to send any
+ more WRITE_READY or WRITE_COMPLETE events. For this simplistic
+ transformation that means we're done. In a more complex
+ transformation we might have to finish writing the transformed
+ data to our output connection. */
+ if (!TSVIOBufferGet(write_vio)) {
+ TSDebug("gzip", "gzip_transform_do->!TSVIOBufferGet(write_vio)...\n");
+ gzip_transform_finish(data);
+
+ TSVIONBytesSet(data->output_vio, data->output_length);
+ TSDebug("gzip-transform", "Compressed size %d (bytes)", data->output_length);
+
+ if (data->output_length > length) {
+ TSDebug("gzip", "gzip_transform_do->!reeanble data->output_vio\n");
+ TSVIOReenable(data->output_vio);
+ }
+ return;
+ }
+
+ /* Determine how much data we have left to read. For this gzip
+ transform plugin this is also the amount of data we have left
+ to write to the output connection. */
+ towrite = TSVIONTodoGet(write_vio);
+ TSDebug("gzip-transform", "gzip_transform_do: towrite: %d",towrite);
+ if (towrite > 0) {
+ /* The amount of data left to read needs to be truncated by
+ the amount of data actually in the read buffer. */
+ avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio));
+ TSDebug("gzip-transform", "gzip_transform_do: avail: %d",avail);
+ if (towrite > avail) {
+ towrite = avail;
+ }
+
+ if (towrite > 0) {
+ TSDebug("gzip-transform", "gzip_transform_do: call gzip_transform_one");
+ gzip_transform_one(data, TSVIOReaderGet(write_vio), towrite);
+ TSDebug("gzip-transform", "gzip_transform_do-> gzip_transform_one finished");
+ /* Modify the write vio to reflect how much data we've
+ completed. */
+ TSVIONDoneSet(write_vio, TSVIONDoneGet(write_vio) + towrite);
+ TSDebug("gzip-transform", "gzip_transform_do-> TSVIONDoneSet finished");
+ }
+ }
+
+ TSDebug("gzip","TSVIONTodoGet(write_vio) -> %d", TSVIONTodoGet(write_vio) );
+
+ /* Now we check the write vio to see if there is data left to
+ read. */
+ if (TSVIONTodoGet(write_vio) > 0) {
+ TSDebug("gzip-transform", "gzip_transform_do-> towrite: %d",towrite);
+ if (towrite > 0) {
+ /* If we output some data then we reenable the output
+ connection by reenabling the output vio. This will wakeup
+ the output connection and allow it to consume data from the
+ output buffer. */
+
+ TSDebug("gzip-transform", "gzip_transform_do: data->output_length > length? %d > %d",data->output_length, length);
+ if (data->output_length > length) {
+ TSDebug("gzip-transform", "gzip_transform_do-> vio renable");
+ TSVIOReenable(data->output_vio);
+ //return;
+ }
+
+ /* Call back the write vio continuation to let it know that we
+ are ready for more data. */
+ TSDebug("gzip-transform", "gzip_transform_do: TSContCall - TS_EVENT_VCONN_WRITE_READY");
+ TSContCall(TSVIOContGet(write_vio), TS_EVENT_VCONN_WRITE_READY, write_vio);
+ }
+ } else {
+ TSDebug("gzip-transform", "gzip_transform_do-> towrite <= 0");
+ /* If there is no data left to read, then we modify the output
+ vio to reflect how much data the output connection should
+ expect. This allows the output connection to know when it
+ is done reading. We then reenable the output connection so
+ that it can consume the data we just gave it. */
+
+ TSDebug("gzip-transform", "gzip_transform_do-> pre gzip_transform_finish");
+ gzip_transform_finish(data);
+ TSDebug("gzip-transform", "gzip_transform_do-> post gzip_transform_finish");
+ TSVIONBytesSet(data->output_vio, data->output_length);
+ TSDebug("gzip-transform", "gzip_transform_do-> post TSVIONBytesSet");
+ TSDebug("gzip-transform", "gzip_transform_do-> Compressed size %d (bytes)", data->output_length);
+
+ if (data->output_length > length) {
+ TSDebug("gzip-transform", "gzip_transform_do-> call TSVIOReenable");
+ TSVIOReenable(data->output_vio);
+ }
+
+ /* Call back the write vio continuation to let it know that we
+ have completed the write operation. */
+ TSDebug("gzip-transform", "gzip_transform_do: TSContCall - TS_EVENT_VCONN_WRITE_COMPLETE");
+ TSContCall(TSVIOContGet(write_vio), TS_EVENT_VCONN_WRITE_COMPLETE, write_vio);
+ }
+ TSDebug("gzip-transform", "gzip_transform_do-> survived");
+}
+
+
+static int
+gzip_transform(TSCont contp, TSEvent event, void *edata)
+{
+ TSDebug("gzip", "gzip_transform event %d\n",event);
+ /* Check to see if the transformation has been closed by a call to
+ TSVConnClose. */
+ if (TSVConnClosedGet(contp)) {
+ TSDebug("gzip","gzip_transform -> transformation has been closed");
+ //.TSHttpTxnRespCacheableSet((TSHttpTxn)edata,0);
+ gzip_data_destroy(TSContDataGet(contp));
+ TSContDestroy(contp);
+ return 0;
+ } else {
+ TSDebug("gzip", "gzip_transform: switch on event\n");
+ switch (event) {
+ case TS_EVENT_ERROR:
+ {
+ TSDebug("gzip", "gzip_transform: TS_EVENT_ERROR starts\n");
+ TSVIO write_vio;
+
+ /* Get the write vio for the write operation that was
+ performed on ourself. This vio contains the continuation of
+ our parent transformation. */
+ write_vio = TSVConnWriteVIOGet(contp);
+
+ /* Call back the write vio continuation to let it know that we
+ have completed the write operation. */
+ TSContCall(TSVIOContGet(write_vio), TS_EVENT_ERROR, write_vio);
+ }
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ TSDebug("gzip", "gzip_transform: TS_EVENT_VCONN_WRITE_COMPLETE starts\n");
+ /* When our output connection says that it has finished
+ reading all the data we've written to it then we should
+ shutdown the write portion of its connection to
+ indicate that we don't want to hear about it anymore. */
+ TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
+ break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ TSDebug("gzip", "gzip_transform: TS_EVENT_VCONN_WRITE_READY starts\n");
+ /* If we get a WRITE_READY we'll attempt to transform more data. */
+ gzip_transform_do(contp);
+ break;
+ case TS_EVENT_IMMEDIATE:
+ TSDebug("gzip", "gzip_transform: TS_EVENT_IMMEDIATE received. how about spitting out an error?");
+ gzip_transform_do(contp);
+ break;
+ default:
+ TSDebug("gzip", "gzip_transform: default starts %d\n", event);
+ /* If we get any other type of event (sent, perhaps, because we were reenabled) then
+ we'll attempt to transform more data. */
+ gzip_transform_do(contp);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+
+static int gzip_transformable(TSHttpTxn txnp, int server)
+{
+ {
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+ TSHttpStatus resp_status;
+ TSMLoc con_field;
+ int con_len;
+ const char * con_val;
+ if (server) {
+ TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
+ } else {
+ TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
+ }
+ resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
+
+ con_field = TSMimeHdrFieldFind(bufp,hdr_loc, TS_MIME_FIELD_CONNECTION,-1);
+ if (con_field) {
+ con_val = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, con_field, 0, &con_len);
+ TSHandleMLocRelease(bufp, hdr_loc, con_field);
+
+ //OS: !!!!!! FIXME !!!!!!!
+ //this is a hotfix for some weird behavior from an origin
+ //needs to be patched properly. this disables support for transactions that send the connection:close header
+ if (con_val && con_len == 5)
+ {
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return -999;
+ }
+ }
+
+ TSDebug("gzip","Got status %d",resp_status);
+ if (TS_HTTP_STATUS_OK == resp_status) {
+ if (TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc) != TS_SUCCESS) {
+ TSError("Unable to release handle to server request");
+ }
+ //return 1;
+ } else {
+ if (TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc) != TS_SUCCESS) {
+ TSError("Unable to release handle to server request");
+ }
+ return -100;
+ }
+ }
+
+
+
+// TSDebug("gzip", "gzip_transformable\n");
+ /* Server response header */
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+ TSMLoc field_loc;
+
+ /* Client request header */
+ TSMBuffer cbuf;
+ TSMLoc chdr;
+ TSMLoc cfield;
+
+ const char *value;
+ int nvalues;
+ int i, deflate_flag, len;
+
+
+ TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
+
+
+ /* check if client accepts "deflate" */
+ cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, -1);
+
+ if (TS_NULL_MLOC != cfield) {
+ nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
+
+ value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, 0, &len);
+ deflate_flag = 0;
+ i = 0;
+ while (nvalues > 0) {
+ if (value && (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0)) {
+ deflate_flag = 1;
+ break;
+ }
+ i++;
+
+ value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len);
+
+ nvalues--;
+ }
+
+ if (!deflate_flag) {
+ TSHandleMLocRelease(cbuf, chdr, cfield);
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+ return -7;
+ }
+
+ TSHandleMLocRelease(cbuf, chdr, cfield);
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+ } else {
+
+ TSHandleMLocRelease(cbuf, chdr, cfield);
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
+ return -6;
+ }
+
+ if (server) {
+ TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
+ } else {
+ TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
+ }
+
+ /* If there already exists a content encoding then we don't want
+ to do anything. */
+ field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
+ if (field_loc) {
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return -3;
+ }
+
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+
+ /* We only want to do gzip compression on documents that have a
+ content type of "text/" or "application/x-javascript". */
+
+ field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
+ if (!field_loc) {
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ return -4;
+ }
+
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len);
+ /*os: FIXME -> vary:accept-encoding needs to be added if any of these contenttypes is encountered
+ */
+ if ( len >= 5 && value && (strncasecmp(value, "text/", sizeof("text/") - 1) == 0)) {
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ TSDebug("gzip-transform", "@ transformable, it is text/*");
+ return 0;
+ } else if ( len >= (sizeof("application/x-javascript") -1) && value && (strncasecmp(value, "application/x-javascript", (sizeof("application/x-javascript") - 1)) == 0)) {
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ TSDebug("gzip-transform", "@ transformable, it is application/x-javascript");
+ return 0;
+ } else {
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ TSDebug("gzip-transform", "@ not a transformable content type");
+ return -5;
+ }
+}
+
+
+static void gzip_transform_add(TSHttpTxn txnp, int server)
+{
+ TSVConn connp;
+ GzipData *data;
+ TSDebug("gzip","zip_transform_add -> tstransformcreate()");
+ connp = TSTransformCreate(gzip_transform, txnp);
+
+ TSDebug("gzip","zip_transform_add -> gzip_data_alloc()");
+ data = gzip_data_alloc();
+ data->txn = txnp;
+
+ TSDebug("gzip","zip_transform_add -> TSContDataSet()");
+ TSContDataSet(connp, data);
+
+ TSDebug("gzip","zip_transform_add -> TSHttpTxnHookAdd()");
+ TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
+}
+
+static void
+normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc)
+{
+// const char * header = "Accept-Encoding";
+// int header_len = strlen(header);
+ TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING
+ , TS_MIME_LEN_ACCEPT_ENCODING);
+
+ int deflate = 0;
+
+ //remove the accept encoding field(s),
+ //while finding out if deflate is supported.
+
+ while (field)
+ {
+ TSMLoc tmp;
+
+ if (!deflate) {
+ int value_count = TSMimeHdrFieldValuesCount(reqp,hdr_loc,field);
+
+ while(value_count > 0) {
+ value_count--;
+
+ int val_len = 0;
+ const char* val = NULL;
+
+ val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
+
+ //FIXME
+ //OS: these aren't always NULL terminated.
+ if (val_len >= 7)//valgrind complains on the strstrs
+ deflate = strstr(val,"deflate") != NULL;
+
+ if(deflate)
+ break;
+ }
+ }
+
+ tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field);
+ TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval?
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ field = tmp;
+ }
+
+ //if deflate is supported,
+ //append a new accept-encoding field in the header
+ if (deflate){
+ TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
+ TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING
+ ,TS_MIME_LEN_ACCEPT_ENCODING);
+ TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1,
+ "deflate", strlen("deflate"));
+ TSMimeHdrFieldAppend(reqp, hdr_loc, field);
+ TSHandleMLocRelease(reqp, hdr_loc, field);
+ }
+}
+
+
+static int
+transform_plugin(TSCont contp, TSEvent event, void *edata)
+{
+ TSHttpTxn txnp = (TSHttpTxn) edata;
+ int reason;
+
+ TSDebug("gzip-transform", "@ transform_plugin()");
+
+ switch (event) {
+ case TS_EVENT_HTTP_READ_REQUEST_HDR:{
+ TSMBuffer req_buf;
+ TSMLoc req_loc;
+ if ( TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS ){
+ normalize_accept_encoding(txnp, req_buf, req_loc);
+ TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+
+ } break;
+ case TS_EVENT_HTTP_READ_RESPONSE_HDR:
+ reason = gzip_transformable(txnp, 1);
+ if (reason >= 0) {
+ TSDebug("gzip-transform", "server content transformable");
+ gzip_transform_add(txnp, 1);
+ } else {
+ TSDebug("gzip-transform", "server content NOT transformable [%d]", reason);
+ }
+
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ break;
+
+ case TS_EVENT_HTTP_READ_CACHE_HDR:
+
+ reason = gzip_transformable(txnp, 0);
+ if (reason >= 0) {
+ TSDebug("gzip-transform", "cached content transformable");
+ gzip_transform_add(txnp, 1);
+ } else {
+ TSDebug("gzip-transform", "cached data: forwarding unchanged (%d)", reason);
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ break;
+
+ default:
+ TSError("gzip transform unknown event, exit!");
+ exit(1);
+ }
+
+ return 0;
+}
+
+
+void
+TSPluginInit(int argc, const char *argv[])
+{
+ dictId = adler32(0L, Z_NULL, 0);
+ if (argc == 2) {
+ strcpy(preload_file, argv[1]);
+ preload = 1;
+ load_dictionary(dictionary, &dictId);
+ }
+
+ TSDebug("gzip-transform", "gzip plugin loads");
+
+ TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(transform_plugin, NULL));
+ TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(transform_plugin, NULL));
+ //TSHttpHookAdd(TS_HTTP_READ_CACHE_HDR_HOOK, TSContCreate(transform_plugin, NULL));
+}
+