You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2017/05/03 14:04:11 UTC
[trafficserver] 01/03: brotli support
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 7.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 62631ffb16ddc597b6c0003f9b4b181e7dbd0101
Author: myraid <sa...@gmail.com>
AuthorDate: Tue Mar 7 15:52:20 2017 -0800
brotli support
remove enfore-brotli from readme
(cherry picked from commit 39d045ecd4037ac2b8d62944c9a08d20667a23c5)
Conflicts:
plugins/gzip/README
---
configure.ac | 6 +
doc/admin-guide/plugins/gzip.en.rst | 24 +++
plugins/gzip/Makefile.inc | 4 +
plugins/gzip/README | 21 +-
plugins/gzip/configuration.cc | 38 +++-
plugins/gzip/configuration.h | 18 +-
plugins/gzip/gzip.cc | 412 ++++++++++++++++++++++++++----------
plugins/gzip/misc.cc | 12 +-
plugins/gzip/misc.h | 30 ++-
9 files changed, 442 insertions(+), 123 deletions(-)
diff --git a/configure.ac b/configure.ac
index 7b1a897..66e71b6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -444,6 +444,12 @@ AC_ARG_WITH([max-threads-per-type],
)
AC_SUBST(max_threads_per_type)
+# Check Brotli
+AC_CHECK_HEADERS([brotli/encode.h], [has_brotli=1],[has_brotli=0])
+AC_CHECK_LIB([brotlienc],[BrotliEncoderCreateInstance],[AC_SUBST([LIB_BROTLIENC],["-lbrotlienc"])],[has_brotli=0])
+AC_SUBST(has_brotli)
+AM_CONDITIONAL([HAS_BROTLI], [ test "x${has_brotli}" = "x1" ])
+
#
# Experimental plugins
#
diff --git a/doc/admin-guide/plugins/gzip.en.rst b/doc/admin-guide/plugins/gzip.en.rst
index 5ba8363..cbf52be 100644
--- a/doc/admin-guide/plugins/gzip.en.rst
+++ b/doc/admin-guide/plugins/gzip.en.rst
@@ -141,6 +141,14 @@ will leave the header intact if the client provided it.
- For when the proxy parses responses, and the resulting compression and
decompression is wasteful.
+supported-algorithms
+----------------------
+
+Provides the compression algorithms that are supported. This will allow the proxy to selectively
+support certain compressions. The default is gzip. Multiple algorthims can be selected using ',' delimiter
+
+-- To selectively support only certain compression algorithms.
+
Examples
========
@@ -162,6 +170,22 @@ might create a configuration with the following options::
disallow /notthis/*.js
flush true
+ # Allows brotli encoded response from origin but is not capable of brotli compression
+ [brotli.allowed.com]
+ enabled true
+ compressible-content-type text/*
+ compressible-content-type application/json
+ flush true
+ supported-algorithms gzip,deflate
+
+ # Supports brotli compression
+ [brotli.compress.com]
+ enabled true
+ compressible-content-type text/*
+ compressible-content-type application/json
+ flush true
+ supported-algorithms br, gzip
+
# This origin does it all
[bar.example.com]
enabled false
diff --git a/plugins/gzip/Makefile.inc b/plugins/gzip/Makefile.inc
index 3cd845c..8762e21 100644
--- a/plugins/gzip/Makefile.inc
+++ b/plugins/gzip/Makefile.inc
@@ -16,3 +16,7 @@
pkglib_LTLIBRARIES += gzip/gzip.la
gzip_gzip_la_SOURCES = gzip/gzip.cc gzip/configuration.cc gzip/misc.cc
+
+gzip_gzip_la_LDFLAGS = \
+ $(AM_LDFLAGS) $(LIB_BROTLIENC)
+
diff --git a/plugins/gzip/README b/plugins/gzip/README
index 4872369..b687cad 100644
--- a/plugins/gzip/README
+++ b/plugins/gzip/README
@@ -9,7 +9,7 @@ make && sudo make install
if no makefile is present, you can compile it using
tsxs -o gzip.so *.cc
-and then install it using
+and then install it using
tsxs -i -o gzip.so
after installation, add a line to plugin.config:
@@ -43,7 +43,12 @@ a sample configuration (sample.gzip.config):
#
# compressible-content-type: wildcard pattern for matching compressible content types
#
-# disallow: wildcard pattern for disablign compression on urls
+# disallow: wildcard pattern for disabling compression on urls
+#
+# allow: wildcard pattern for enabling compression on urls
+#
+# supported-algorithms: compression algorithms supported. comma separated algorithms. Default is gzip
+#
######################################################################
#first, we configure the default/global plugin behaviour
@@ -60,7 +65,14 @@ disallow /notthis/*.js
disallow /notthat*
disallow */bla*
-#override the global configuration for a host.
+allow */here/*
+#disabling is possible too. trying to deprecate disallow
+allow !*/nothere/*
+
+#supported algorithms
+supported-algorithms br,gzip
+
+#override the global configuration for a host.
#www.foo.nl does NOT inherit anything
[www.foo.nl]
enabled true
@@ -70,3 +82,6 @@ compressible-content-type text/*
cache false
disallow /notthis/*.js
disallow /notthat*
+
+allow /this/*.js
+allow !/notthat/*.css
diff --git a/plugins/gzip/configuration.cc b/plugins/gzip/configuration.cc
index 80d8cf4..e0101a0 100644
--- a/plugins/gzip/configuration.cc
+++ b/plugins/gzip/configuration.cc
@@ -25,6 +25,7 @@
#include <fstream>
#include <algorithm>
#include <vector>
+#include <sstream>
#include <fnmatch.h>
namespace Gzip
@@ -93,7 +94,8 @@ enum ParserState {
kParseEnable,
kParseCache,
kParseDisallow,
- kParseFlush
+ kParseFlush,
+ kParseAlgorithms
};
void
@@ -181,6 +183,34 @@ HostConfiguration::is_content_type_compressible(const char *content_type, int co
return is_match;
}
+void
+HostConfiguration::add_compression_algorithms(const string &algorithms)
+{
+ istringstream compress_algo(algorithms);
+ string token;
+ compression_algorithms_ = ALGORITHM_DEFAULT; // remove the default gzip.
+ while (getline(compress_algo, token, ',')) {
+ if (token.find("br") != string::npos) {
+#ifdef HAVE_BROTLI_ENCODE_H
+ compression_algorithms_ |= ALGORITHM_BROTLI;
+#else
+ error("supported-algorithms: brotli support not compiled in.");
+#endif
+ } else if (token.find("gzip") != string::npos)
+ compression_algorithms_ |= ALGORITHM_GZIP;
+ else if (token.find("deflate") != string::npos)
+ compression_algorithms_ |= ALGORITHM_DEFLATE;
+ else
+ error("Unknown compression type. Supported compression-algorithms <br,gzip,deflate>.");
+ }
+}
+
+int
+HostConfiguration::compression_algorithms()
+{
+ return compression_algorithms_;
+}
+
Configuration *
Configuration::Parse(const char *path)
{
@@ -263,6 +293,8 @@ Configuration::Parse(const char *path)
state = kParseDisallow;
} else if (token == "flush") {
state = kParseFlush;
+ } else if (token == "supported-algorithms") {
+ state = kParseAlgorithms;
} else {
warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno);
}
@@ -291,6 +323,10 @@ Configuration::Parse(const char *path)
current_host_configuration->set_flush(token == "true");
state = kParseStart;
break;
+ case kParseAlgorithms:
+ current_host_configuration->add_compression_algorithms(token);
+ state = kParseStart;
+ break;
}
}
}
diff --git a/plugins/gzip/configuration.h b/plugins/gzip/configuration.h
index 04ff023..09c9559 100644
--- a/plugins/gzip/configuration.h
+++ b/plugins/gzip/configuration.h
@@ -33,11 +33,24 @@ namespace Gzip
{
typedef std::vector<std::string> StringContainer;
+enum CompressionAlgorithm {
+ ALGORITHM_DEFAULT = 0,
+ ALGORITHM_DEFLATE = 1,
+ ALGORITHM_GZIP = 2,
+ ALGORITHM_BROTLI = 4 // For bit manipulations
+};
+
class HostConfiguration
{
public:
explicit HostConfiguration(const std::string &host)
- : host_(host), enabled_(true), cache_(true), remove_accept_encoding_(false), flush_(false), ref_count_(0)
+ : host_(host),
+ enabled_(true),
+ cache_(true),
+ remove_accept_encoding_(false),
+ flush_(false),
+ compression_algorithms_(ALGORITHM_GZIP),
+ ref_count_(0)
{
}
@@ -96,6 +109,8 @@ public:
void add_compressible_content_type(const std::string &content_type);
bool is_url_allowed(const char *url, int url_len);
bool is_content_type_compressible(const char *content_type, int content_type_length);
+ void add_compression_algorithms(const std::string &algorithms);
+ int compression_algorithms();
// Ref-counting these host configuration objects
void
@@ -118,6 +133,7 @@ private:
bool cache_;
bool remove_accept_encoding_;
bool flush_;
+ int compression_algorithms_;
volatile int ref_count_;
StringContainer compressible_content_types_;
diff --git a/plugins/gzip/gzip.cc b/plugins/gzip/gzip.cc
index 51110b1..ad6bbe0 100644
--- a/plugins/gzip/gzip.cc
+++ b/plugins/gzip/gzip.cc
@@ -24,6 +24,10 @@
#include <string.h>
#include <zlib.h>
+#if HAVE_BROTLI_ENCODE_H
+#include <brotli/encode.h>
+#endif
+
#include "ts/ts.h"
#include "ts/ink_defs.h"
@@ -36,7 +40,6 @@ using namespace std;
using namespace Gzip;
// FIXME: custom dictionaries would be nice. configurable/content-type?
-// FIXME: look into autoscaling the compression level based on connection speed
// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith
// might be served better with little or no compression at all
// FIXME: look into compressing from the task thread pool
@@ -50,37 +53,48 @@ using namespace Gzip;
const int ZLIB_COMPRESSION_LEVEL = 6;
const char *global_hidden_header_name;
-const char *dictionary = nullptr;
+const char *dictionary = nullptr;
+const char *TS_HTTP_VALUE_BROTLI = "br";
+const int TS_HTTP_LEN_BROTLI = 2;
+
+// brotli compression quality 1-11. Testing proved level '6'
+#if HAVE_BROTLI_ENCODE_H
+const int BROTLI_COMPRESSION_LEVEL = 6;
+const int BROTLI_LGW = 16;
+#endif
// Current global configuration, and the previous one (for cleanup)
Configuration *cur_config = nullptr;
Configuration *prev_config = nullptr;
-static GzipData *
-gzip_data_alloc(int compression_type)
+static Data *
+data_alloc(int compression_type, int compression_algorithms)
{
- GzipData *data;
+ Data *data;
int err;
- data = (GzipData *)TSmalloc(sizeof(GzipData));
- data->downstream_vio = nullptr;
- data->downstream_buffer = nullptr;
- data->downstream_reader = nullptr;
- data->downstream_length = 0;
- data->state = transform_state_initialized;
- data->compression_type = compression_type;
- data->zstrm.next_in = Z_NULL;
- data->zstrm.avail_in = 0;
- data->zstrm.total_in = 0;
- data->zstrm.next_out = Z_NULL;
- data->zstrm.avail_out = 0;
- data->zstrm.total_out = 0;
- data->zstrm.zalloc = gzip_alloc;
- data->zstrm.zfree = gzip_free;
- data->zstrm.opaque = (voidpf) nullptr;
- data->zstrm.data_type = Z_ASCII;
-
- int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
+ data = (Data *)TSmalloc(sizeof(Data));
+ data->downstream_vio = nullptr;
+ data->downstream_buffer = nullptr;
+ data->downstream_reader = nullptr;
+ data->downstream_length = 0;
+ data->state = transform_state_initialized;
+ data->compression_type = compression_type;
+ data->compression_algorithms = compression_algorithms;
+ data->zstrm.next_in = Z_NULL;
+ data->zstrm.avail_in = 0;
+ data->zstrm.total_in = 0;
+ data->zstrm.next_out = Z_NULL;
+ data->zstrm.avail_out = 0;
+ data->zstrm.total_out = 0;
+ data->zstrm.zalloc = gzip_alloc;
+ data->zstrm.zfree = gzip_free;
+ data->zstrm.opaque = (voidpf) nullptr;
+ data->zstrm.data_type = Z_ASCII;
+
+ int window_bits = WINDOW_BITS_GZIP;
+ if (compression_type & COMPRESSION_TYPE_DEFLATE)
+ window_bits = WINDOW_BITS_DEFLATE;
err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
@@ -94,12 +108,28 @@ gzip_data_alloc(int compression_type)
fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
}
}
-
+#if HAVE_BROTLI_ENCODE_H
+ if (compression_type & COMPRESSION_TYPE_BROTLI) {
+ debug("gzip-transform: brotli compression. Create Brotli Encoder Instance.");
+ data->bstrm.br = BrotliEncoderCreateInstance(0, 0, 0);
+ if (!data->bstrm.br) {
+ fatal("gzip-transform: ERROR: Brotli Encoder Instance Failed");
+ }
+ BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, BROTLI_COMPRESSION_LEVEL);
+ BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, BROTLI_LGW);
+ data->bstrm.next_in = nullptr;
+ data->bstrm.avail_in = 0;
+ data->bstrm.total_in = 0;
+ data->bstrm.next_out = nullptr;
+ data->bstrm.avail_out = 0;
+ data->bstrm.total_out = 0;
+ }
+#endif
return data;
}
static void
-gzip_data_destroy(GzipData *data)
+data_destroy(Data *data)
{
TSReleaseAssert(data);
@@ -111,24 +141,41 @@ gzip_data_destroy(GzipData *data)
TSIOBufferDestroy(data->downstream_buffer);
}
+// brotlidestory
+#if HAVE_BROTLI_ENCODE_H
+ BrotliEncoderDestroyInstance(data->bstrm.br);
+#endif
+
TSfree(data);
}
static TSReturnCode
-gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type)
+content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type, int algorithm)
{
TSReturnCode ret;
TSMLoc ce_loc;
-
+ const char *value = nullptr;
+ int value_len = 0;
// Delete Content-Encoding if present???
+ if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) {
+ value = TS_HTTP_VALUE_BROTLI;
+ value_len = TS_HTTP_LEN_BROTLI;
+ } else if (compression_type & COMPRESSION_TYPE_GZIP && (algorithm & ALGORITHM_GZIP)) {
+ value = TS_HTTP_VALUE_GZIP;
+ value_len = TS_HTTP_LEN_GZIP;
+ } else if (compression_type & COMPRESSION_TYPE_DEFLATE && (algorithm & ALGORITHM_DEFLATE)) {
+ value = TS_HTTP_VALUE_DEFLATE;
+ value_len = TS_HTTP_LEN_DEFLATE;
+ }
+
+ if (value_len == 0) {
+ error("no need to add Content-Encoding header");
+ return TS_SUCCESS;
+ }
if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, TS_MIME_LEN_CONTENT_ENCODING, &ce_loc)) ==
TS_SUCCESS) {
- if (compression_type == COMPRESSION_TYPE_DEFLATE) {
- ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, TS_HTTP_VALUE_DEFLATE, TS_HTTP_LEN_DEFLATE);
- } else if (compression_type == COMPRESSION_TYPE_GZIP) {
- ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP);
- }
+ ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, value, value_len);
if (ret == TS_SUCCESS) {
ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
}
@@ -143,7 +190,7 @@ gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compressi
}
static TSReturnCode
-gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
+vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
{
TSReturnCode ret;
TSMLoc ce_loc;
@@ -186,7 +233,7 @@ gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
// FIXME: the etag alteration isn't proper. it should modify the value inside quotes
// specify a very header..
static TSReturnCode
-gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
+etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
{
TSReturnCode ret = TS_SUCCESS;
TSMLoc ce_loc;
@@ -220,7 +267,7 @@ gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
// FIXME: some things are potentially compressible. those responses
static void
-gzip_transform_init(TSCont contp, GzipData *data)
+compress_transform_init(TSCont contp, Data *data)
{
// update the vary, content-encoding, and etag response headers
// prepare the downstream for transforming
@@ -236,8 +283,8 @@ gzip_transform_init(TSCont contp, GzipData *data)
return;
}
- if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == TS_SUCCESS &&
- gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS && gzip_etag_header(bufp, hdr_loc) == TS_SUCCESS) {
+ if (content_encoding_header(bufp, hdr_loc, data->compression_type, data->compression_algorithms) == TS_SUCCESS &&
+ vary_header(bufp, hdr_loc) == TS_SUCCESS && etag_header(bufp, hdr_loc) == TS_SUCCESS) {
downstream_conn = TSTransformOutputVConnGet(contp);
data->downstream_buffer = TSIOBufferCreate();
data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
@@ -248,14 +295,102 @@ gzip_transform_init(TSCont contp, GzipData *data)
}
static void
-gzip_transform_one(GzipData *data, TSIOBufferReader upstream_reader, int amount)
+gzip_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length)
{
TSIOBufferBlock downstream_blkp;
- const char *upstream_buffer;
char *downstream_buffer;
- int64_t upstream_length, downstream_length;
+ int64_t downstream_length;
+ int err;
+ data->zstrm.next_in = (unsigned char *)upstream_buffer;
+ data->zstrm.avail_in = upstream_length;
+
+ while (data->zstrm.avail_in > 0) {
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+
+ data->zstrm.next_out = (unsigned char *)downstream_buffer;
+ data->zstrm.avail_out = downstream_length;
+
+ if (!data->hc->flush()) {
+ err = deflate(&data->zstrm, Z_NO_FLUSH);
+ } else {
+ err = deflate(&data->zstrm, Z_SYNC_FLUSH);
+ }
+
+ if (err != Z_OK) {
+ warning("deflate() call failed: %d", err);
+ }
+
+ if (downstream_length > data->zstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
+ data->downstream_length += (downstream_length - data->zstrm.avail_out);
+ }
+
+ if (data->zstrm.avail_out > 0) {
+ if (data->zstrm.avail_in != 0) {
+ error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
+ }
+ }
+ }
+}
+
+static void
+brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length)
+{
+#if HAVE_BROTLI_ENCODE_H
+ TSIOBufferBlock downstream_blkp;
+ char *downstream_buffer;
+ int64_t downstream_length;
int err;
+ data->bstrm.avail_in = upstream_length;
+
+ while (data->bstrm.avail_in > 0) {
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+
+ data->bstrm.next_out = (unsigned char *)downstream_buffer;
+ data->bstrm.avail_out = downstream_length;
+ data->bstrm.total_out = 0;
+
+ data->bstrm.next_in = (uint8_t *)upstream_buffer;
+ if (!data->hc->flush()) {
+ err = BrotliEncoderCompressStream(data->bstrm.br, BROTLI_OPERATION_PROCESS, &data->bstrm.avail_in,
+ (const uint8_t **)&data->bstrm.next_in, &data->bstrm.avail_out, &data->bstrm.next_out,
+ &data->bstrm.total_out);
+ } else {
+ err = BrotliEncoderCompressStream(data->bstrm.br, BROTLI_OPERATION_FLUSH, &data->bstrm.avail_in,
+ (const uint8_t **)&data->bstrm.next_in, &data->bstrm.avail_out, &data->bstrm.next_out,
+ &data->bstrm.total_out);
+ }
+
+ if (err != BROTLI_TRUE) {
+ warning("BrotliEncoderCompressStream() call failed: %d", err);
+ }
+
+ if (downstream_length > (int64_t)data->bstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out);
+ data->downstream_length += (downstream_length - data->bstrm.avail_out);
+ }
+
+ if (data->bstrm.avail_out > 0) {
+ if (data->bstrm.avail_in != 0) {
+ error("brotli-transform: ERROR: brotli avail_in is (%lu): should be 0", data->bstrm.avail_in);
+ }
+ }
+ }
+ data->bstrm.total_in += upstream_length;
+#else
+ error("brotli-transform: ERROR: compile with brotli support");
+#endif
+}
+
+static void
+compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount)
+{
+ TSIOBufferBlock downstream_blkp;
+ const char *upstream_buffer;
+ int64_t upstream_length;
while (amount > 0) {
downstream_blkp = TSIOBufferReaderStart(upstream_reader);
if (!downstream_blkp) {
@@ -273,38 +408,13 @@ gzip_transform_one(GzipData *data, TSIOBufferReader upstream_reader, int amount)
upstream_length = amount;
}
- data->zstrm.next_in = (unsigned char *)upstream_buffer;
- data->zstrm.avail_in = upstream_length;
-
- while (data->zstrm.avail_in > 0) {
- downstream_blkp = TSIOBufferStart(data->downstream_buffer);
- downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
-
- data->zstrm.next_out = (unsigned char *)downstream_buffer;
- data->zstrm.avail_out = downstream_length;
-
- if (!data->hc->flush()) {
- debug("gzip_transform: deflate with Z_NO_FLUSH");
- err = deflate(&data->zstrm, Z_NO_FLUSH);
- } else {
- debug("gzip_transform: deflate with Z_SYNC_FLUSH");
- err = deflate(&data->zstrm, Z_SYNC_FLUSH);
- }
-
- if (err != Z_OK) {
- warning("deflate() call failed: %d", err);
- }
-
- if (downstream_length > data->zstrm.avail_out) {
- TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
- data->downstream_length += (downstream_length - data->zstrm.avail_out);
- }
-
- if (data->zstrm.avail_out > 0) {
- if (data->zstrm.avail_in != 0) {
- error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
- }
- }
+ if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) {
+ brotli_transform_one(data, upstream_buffer, upstream_length);
+ } else if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE)) &&
+ (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) {
+ gzip_transform_one(data, upstream_buffer, upstream_length);
+ } else {
+ warning("No compression supported. Shoudn't come here.");
}
TSIOBufferReaderConsume(upstream_reader, upstream_length);
@@ -313,7 +423,7 @@ gzip_transform_one(GzipData *data, TSIOBufferReader upstream_reader, int amount)
}
static void
-gzip_transform_finish(GzipData *data)
+gzip_transform_finish(Data *data)
{
if (data->state == transform_state_output) {
TSIOBufferBlock downstream_blkp;
@@ -350,30 +460,94 @@ gzip_transform_finish(GzipData *data)
if (data->downstream_length != (int64_t)(data->zstrm.total_out)) {
error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length, data->zstrm.total_out);
}
-
+ debug("gzip-transform: Finished gzip");
gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
}
}
static void
-gzip_transform_do(TSCont contp)
+brotli_transform_finish(Data *data)
+{
+#if HAVE_BROTLI_ENCODE_H
+ if (data->state == transform_state_output) {
+ TSIOBufferBlock downstream_blkp;
+ char *downstream_buffer;
+ int64_t downstream_length;
+ int err;
+
+ data->state = transform_state_finished;
+
+ for (;;) {
+ downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+
+ downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+ data->bstrm.next_out = (unsigned char *)downstream_buffer;
+ data->bstrm.avail_out = downstream_length;
+
+ err = BrotliEncoderCompressStream(data->bstrm.br, BROTLI_OPERATION_FINISH, &data->bstrm.avail_in,
+ (const uint8_t **)&data->bstrm.next_in, &data->bstrm.avail_out, &data->bstrm.next_out,
+ &data->bstrm.total_out);
+
+ if (downstream_length > (int64_t)data->bstrm.avail_out) {
+ TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out);
+ data->downstream_length += (downstream_length - data->bstrm.avail_out);
+ }
+ if (!BrotliEncoderIsFinished(data->bstrm.br)) {
+ continue;
+ }
+
+ if (err != BROTLI_TRUE) { /* some more data to encode */
+ warning("brotli_transform: BrotliEncoderCompressStream should return BROTLI_TRUE");
+ }
+
+ break;
+ }
+
+ if (data->downstream_length != (int64_t)(data->bstrm.total_out)) {
+ error("brotli-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length, data->bstrm.total_out);
+ }
+ debug("brotli-transform: Finished brotli");
+ gzip_log_ratio(data->bstrm.total_in, data->downstream_length);
+ }
+#else
+ error("brotli-transform: compile with brotli support");
+#endif
+}
+
+static void
+compress_transform_finish(Data *data)
+{
+ if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) {
+ brotli_transform_finish(data);
+ debug("brotli-transform: Brolti compression finish.");
+ } else if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE)) &&
+ (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) {
+ gzip_transform_finish(data);
+ debug("gzip-transform: Gzip compression finish.");
+ } else {
+ warning("No Compression matched, shouldn't come here.");
+ }
+}
+
+static void
+compress_transform_do(TSCont contp)
{
TSVIO upstream_vio;
- GzipData *data;
+ Data *data;
int64_t upstream_todo;
int64_t upstream_avail;
int64_t downstream_bytes_written;
- data = (GzipData *)TSContDataGet(contp);
+ data = (Data *)TSContDataGet(contp);
if (data->state == transform_state_initialized) {
- gzip_transform_init(contp, data);
+ compress_transform_init(contp, data);
}
upstream_vio = TSVConnWriteVIOGet(contp);
downstream_bytes_written = data->downstream_length;
if (!TSVIOBufferGet(upstream_vio)) {
- gzip_transform_finish(data);
+ compress_transform_finish(data);
TSVIONBytesSet(data->downstream_vio, data->downstream_length);
@@ -393,7 +567,7 @@ gzip_transform_do(TSCont contp)
}
if (upstream_todo > 0) {
- gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
+ compress_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
}
}
@@ -406,7 +580,7 @@ gzip_transform_do(TSCont contp)
TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
}
} else {
- gzip_transform_finish(data);
+ compress_transform_finish(data);
TSVIONBytesSet(data->downstream_vio, data->downstream_length);
if (data->downstream_length > downstream_bytes_written) {
@@ -418,10 +592,10 @@ gzip_transform_do(TSCont contp)
}
static int
-gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
+compress_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
{
if (TSVConnClosedGet(contp)) {
- gzip_data_destroy((GzipData *)TSContDataGet(contp));
+ data_destroy((Data *)TSContDataGet(contp));
TSContDestroy(contp);
return 0;
} else {
@@ -435,14 +609,14 @@ gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
break;
case TS_EVENT_VCONN_WRITE_READY:
- gzip_transform_do(contp);
+ compress_transform_do(contp);
break;
case TS_EVENT_IMMEDIATE:
- gzip_transform_do(contp);
+ compress_transform_do(contp);
break;
default:
warning("unknown event [%d]", event);
- gzip_transform_do(contp);
+ compress_transform_do(contp);
break;
}
}
@@ -451,7 +625,7 @@ gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
}
static int
-gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration, int *compress_type)
+transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration, int *compress_type, int *algorithms)
{
/* Server response header */
TSMBuffer bufp;
@@ -468,6 +642,13 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
int i, compression_acceptable, len;
TSHttpStatus resp_status;
+ /*
+ // Before anything, check atleast one compression algorithm is supported
+ if (host_configuration->compression_algorithms() == ALGORITHM_DEFAULT) {
+ info("No compression algorithms configured");
+ return 0;
+ }
+ */
if (server) {
if (TS_SUCCESS != TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc)) {
return 0;
@@ -503,7 +684,8 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
return 0;
}
- cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+ *algorithms = host_configuration->compression_algorithms();
+ cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
if (cfield != TS_NULL_MLOC) {
compression_acceptable = 0;
nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
@@ -513,14 +695,18 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
continue;
}
- if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
- compression_acceptable = 1;
- *compress_type = COMPRESSION_TYPE_DEFLATE;
- break;
+ if (strncasecmp(value, "br", sizeof("br") - 1) == 0) {
+ if (*algorithms & ALGORITHM_BROTLI)
+ compression_acceptable = 1;
+ *compress_type |= COMPRESSION_TYPE_BROTLI;
+ } else if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
+ if (*algorithms & ALGORITHM_DEFLATE)
+ compression_acceptable = 1;
+ *compress_type |= COMPRESSION_TYPE_DEFLATE;
} else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) {
- compression_acceptable = 1;
- *compress_type = COMPRESSION_TYPE_GZIP;
- break;
+ if (*algorithms & ALGORITHM_GZIP)
+ compression_acceptable = 1;
+ *compress_type |= COMPRESSION_TYPE_GZIP;
}
}
@@ -528,7 +714,7 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
if (!compression_acceptable) {
- info("no acceptable encoding found in request header, not compressible");
+ info("no acceptable encoding match found in request header, not compressible");
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
return 0;
}
@@ -574,21 +760,24 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
}
static void
-gzip_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type)
+compress_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type, int algorithms)
{
TSVConn connp;
- GzipData *data;
+ Data *data;
TSHttpTxnUntransformedRespCache(txnp, 1);
if (!hc->cache()) {
+ debug("TransformedRespCache not enabled");
TSHttpTxnTransformedRespCache(txnp, 0);
} else {
+ debug("TransformedRespCache enabled");
+ TSHttpTxnUntransformedRespCache(txnp, 0);
TSHttpTxnTransformedRespCache(txnp, 1);
}
- connp = TSTransformCreate(gzip_transform, txnp);
- data = gzip_data_alloc(compress_type);
+ connp = TSTransformCreate(compress_transform, txnp);
+ data = data_alloc(compress_type, algorithms);
data->txn = txnp;
data->hc = hc;
@@ -620,7 +809,8 @@ static int
transform_plugin(TSCont contp, TSEvent event, void *edata)
{
TSHttpTxn txnp = (TSHttpTxn)edata;
- int compress_type = COMPRESSION_TYPE_DEFLATE;
+ int compress_type = COMPRESSION_TYPE_DEFAULT;
+ int algorithms = ALGORITHM_DEFAULT;
HostConfiguration *hc = (HostConfiguration *)TSContDataGet(contp);
switch (event) {
@@ -639,8 +829,8 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
}
}
- if (gzip_transformable(txnp, true, hc, &compress_type)) {
- gzip_transform_add(txnp, hc, compress_type);
+ if (transformable(txnp, true, hc, &compress_type, &algorithms)) {
+ compress_transform_add(txnp, hc, compress_type, algorithms);
}
}
break;
@@ -667,8 +857,8 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
if (TS_ERROR != TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) && (TS_CACHE_LOOKUP_HIT_FRESH == obj_status)) {
if (hc != nullptr) {
info("handling compression of cached object");
- if (gzip_transformable(txnp, false, hc, &compress_type)) {
- gzip_transform_add(txnp, hc, compress_type);
+ if (transformable(txnp, false, hc, &compress_type, &algorithms)) {
+ compress_transform_add(txnp, hc, compress_type, algorithms);
}
}
} else {
@@ -703,7 +893,7 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
* further processing
*/
static void
-handle_gzip_request(TSHttpTxn txnp, Configuration *config)
+handle_request(TSHttpTxn txnp, Configuration *config)
{
TSMBuffer req_buf;
TSMLoc req_loc;
@@ -721,14 +911,12 @@ handle_gzip_request(TSHttpTxn txnp, Configuration *config)
if (hc->has_disallows()) {
int url_len;
char *url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
-
- allowed = hc->is_url_allowed(url, url_len);
+ allowed = hc->is_url_allowed(url, url_len);
TSfree(url);
} else {
allowed = true;
}
}
-
if (allowed) {
TSCont transform_contp = TSContCreate(transform_plugin, nullptr);
@@ -753,7 +941,7 @@ transform_global_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edat
switch (event) {
case TS_EVENT_HTTP_READ_REQUEST_HDR:
// Handle gzip request and use the global configs
- handle_gzip_request(txnp, nullptr);
+ handle_request(txnp, nullptr);
break;
default:
@@ -887,7 +1075,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo *rri)
info("Remap Rules configured for gzip");
Configuration *config = (Configuration *)instance;
// Handle gzip request and use the configs populated from remap instance
- handle_gzip_request(txnp, config);
+ handle_request(txnp, config);
}
return TSREMAP_NO_REMAP;
}
diff --git a/plugins/gzip/misc.cc b/plugins/gzip/misc.cc
index 161abda..c0d750e 100644
--- a/plugins/gzip/misc.cc
+++ b/plugins/gzip/misc.cc
@@ -47,7 +47,7 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo
TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
int deflate = 0;
int gzip = 0;
-
+ int br = 0;
// remove the accept encoding field(s),
// while finding out if gzip or deflate is supported.
while (field) {
@@ -63,6 +63,9 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo
--value_count;
val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
+ if (val_len == (int)strlen("br")) {
+ br = !strncmp(val, "br", val_len);
+ }
if (val_len == (int)strlen("gzip")) {
gzip = !strncmp(val, "gzip", val_len);
} else if (val_len == (int)strlen("deflate")) {
@@ -78,10 +81,13 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo
}
// append a new accept-encoding field in the header
- if (deflate || gzip) {
+ if (deflate || gzip || br) {
TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
-
+ if (br) {
+ TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "br", strlen("br"));
+ info("normalized accept encoding to br");
+ }
if (gzip) {
TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
info("normalized accept encoding to gzip");
diff --git a/plugins/gzip/misc.h b/plugins/gzip/misc.h
index 60f5aea..6c00cc0 100644
--- a/plugins/gzip/misc.h
+++ b/plugins/gzip/misc.h
@@ -29,6 +29,10 @@
#include <stdlib.h>
#include <stdio.h>
+#if HAVE_BROTLI_ENCODE_H
+#include <brotli/encode.h>
+#endif
+
#include "configuration.h"
using namespace Gzip;
@@ -39,8 +43,12 @@ static const int WINDOW_BITS_DEFLATE = -15;
static const int WINDOW_BITS_GZIP = 31;
// misc
-static const int COMPRESSION_TYPE_DEFLATE = 1;
-static const int COMPRESSION_TYPE_GZIP = 2;
+enum CompressionType {
+ COMPRESSION_TYPE_DEFAULT = 0,
+ COMPRESSION_TYPE_DEFLATE = 1,
+ COMPRESSION_TYPE_GZIP = 2,
+ COMPRESSION_TYPE_BROTLI = 4
+};
// this one is used to rename the accept encoding header
// it will be restored later on
@@ -53,6 +61,18 @@ enum transform_state {
transform_state_finished,
};
+#if HAVE_BROTLI_ENCODE_H
+typedef struct {
+ BrotliEncoderState *br;
+ uint8_t *next_in;
+ size_t avail_in;
+ uint8_t *next_out;
+ size_t avail_out;
+ size_t total_in;
+ size_t total_out;
+} b_stream;
+#endif
+
typedef struct {
TSHttpTxn txn;
HostConfiguration *hc;
@@ -63,7 +83,11 @@ typedef struct {
z_stream zstrm;
enum transform_state state;
int compression_type;
-} GzipData;
+ int compression_algorithms;
+#if HAVE_BROTLI_ENCODE_H
+ b_stream bstrm;
+#endif
+} Data;
voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
void gzip_free(voidpf opaque, voidpf address);
--
To stop receiving notification emails like this one, please contact
"commits@trafficserver.apache.org" <co...@trafficserver.apache.org>.