You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by bc...@apache.org on 2020/11/30 18:33:54 UTC
[trafficserver] branch 8.1.x updated: ESI: Ensure gzip header is
always initialized (#7358)
This is an automated email from the ASF dual-hosted git repository.
bcall pushed a commit to branch 8.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/8.1.x by this push:
new 0960ef5 ESI: Ensure gzip header is always initialized (#7358)
0960ef5 is described below
commit 0960ef5b872c218796035462a5d70a40489a26da
Author: Brian Neradt <br...@gmail.com>
AuthorDate: Mon Nov 30 12:33:42 2020 -0600
ESI: Ensure gzip header is always initialized (#7358)
There were certain code paths that could be reached in which
stream_finish could be called without stream_encode having been called.
This would result in the gzip header not being encoded in the stream,
which would produce malformed gzip content.
---
doc/admin-guide/plugins/esi.en.rst | 8 +-
plugins/esi/esi.cc | 9 +-
plugins/esi/lib/EsiGzip.cc | 19 +-
plugins/esi/lib/EsiGzip.h | 27 +-
tests/gold_tests/pluginTest/esi/esi.test.py | 310 +++++++++++++++++++++
.../pluginTest/esi/gold/empty_response_body.gold | 14 +
tests/gold_tests/pluginTest/esi/gold/esi_body.gold | 12 +
.../pluginTest/esi/gold/esi_gzipped.gold | 10 +
.../pluginTest/esi/gold/esi_headers.gold | 6 +
9 files changed, 399 insertions(+), 16 deletions(-)
diff --git a/doc/admin-guide/plugins/esi.en.rst b/doc/admin-guide/plugins/esi.en.rst
index 1bbb916..b05ea40 100644
--- a/doc/admin-guide/plugins/esi.en.rst
+++ b/doc/admin-guide/plugins/esi.en.rst
@@ -80,11 +80,11 @@ Enabling ESI
2. There are four options you can add to the above.
-- "--private-response" will add private cache control and expires header to the processed ESI document.
-- "--packed-node-support" will enable the support for using packed node, which will improve the performance of parsing
+- ``--private-response`` will add private cache control and expires header to the processed ESI document.
+- ``--packed-node-support`` will enable the support for using packed node, which will improve the performance of parsing
cached ESI document.
-- "--disable-gzip-output" will disable gzipped output, which will NOT gzip the output anyway.
-- "--first-byte-flush" will enable the first byte flush feature, which will flush content to users as soon as the entire
+- ``--disable-gzip-output`` will disable gzipped output, which will NOT gzip the output anyway.
+- ``--first-byte-flush`` will enable the first byte flush feature, which will flush content to users as soon as the entire
ESI document is received and parsed without all ESI includes fetched (the flushing will stop at the ESI include markup
till that include is fetched).
diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc
index d6d0301..debc557 100644
--- a/plugins/esi/esi.cc
+++ b/plugins/esi/esi.cc
@@ -776,7 +776,7 @@ transformData(TSCont contp)
out_data_len = 0;
out_data = "";
} else {
- TSDebug(cont_data->debug_tag, "[%s] Compressed document from size %d to %d bytes", __FUNCTION__, out_data_len,
+ TSDebug(cont_data->debug_tag, "[%s] Compressed document from size %d to %d bytes via gzip", __FUNCTION__, out_data_len,
(int)cdata.size());
out_data_len = cdata.size();
out_data = cdata.data();
@@ -841,12 +841,9 @@ transformData(TSCont contp)
if (!cont_data->esi_gzip->stream_encode(out_data, cdata)) {
TSError("[esi][%s] Error while gzipping content", __FUNCTION__);
} else {
- TSDebug(cont_data->debug_tag, "[%s] Compressed document from size %d to %d bytes", __FUNCTION__, (int)out_data.size(),
- (int)cdata.size());
+ TSDebug(cont_data->debug_tag, "[%s] Compressed document from size %d to %d bytes via EsiGzip", __FUNCTION__,
+ (int)out_data.size(), (int)cdata.size());
}
- }
-
- if (cont_data->gzip_output) {
if (TSIOBufferWrite(TSVIOBufferGet(cont_data->output_vio), cdata.data(), cdata.size()) == TS_ERROR) {
TSError("[esi][%s] Error while writing bytes to downstream VC", __FUNCTION__);
return 0;
diff --git a/plugins/esi/lib/EsiGzip.cc b/plugins/esi/lib/EsiGzip.cc
index 6c0c7cf..65f9a15 100644
--- a/plugins/esi/lib/EsiGzip.cc
+++ b/plugins/esi/lib/EsiGzip.cc
@@ -30,7 +30,7 @@ using std::string;
using namespace EsiLib;
EsiGzip::EsiGzip(const char *debug_tag, ComponentBase::Debug debug_func, ComponentBase::Error error_func)
- : ComponentBase(debug_tag, debug_func, error_func), _downstream_length(0), _total_data_length(0)
+ : ComponentBase(debug_tag, debug_func, error_func), _downstream_length(0), _total_data_length(0), _crc(0)
{
// Zlib _zstrm varibles are initialized when they are required in runDeflateLoop
// coverity[uninit_member]
@@ -71,6 +71,7 @@ runDeflateLoop(z_stream &zstrm, int flush, std::string &cdata)
bool
EsiGzip::stream_encode(const char *data, int data_len, std::string &cdata)
{
+ const auto initial_cdata_size = cdata.size();
if (_downstream_length == 0) {
cdata.assign(GZIP_HEADER_SIZE, 0); // reserving space for the header
cdata[0] = MAGIC_BYTE_1;
@@ -111,10 +112,9 @@ EsiGzip::stream_encode(const char *data, int data_len, std::string &cdata)
return false;
}
_crc = crc32(_crc, reinterpret_cast<const Bytef *>(data), data_len);
- _downstream_length += cdata.size();
_total_data_length += data_len;
}
-
+ _downstream_length += cdata.size() - initial_cdata_size;
deflateEnd(&_zstrm);
return true;
@@ -123,6 +123,17 @@ EsiGzip::stream_encode(const char *data, int data_len, std::string &cdata)
bool
EsiGzip::stream_finish(std::string &cdata, int &downstream_length)
{
+ if (_downstream_length == 0) {
+ // We need to run encode first to get the gzip header inserted.
+ if (!stream_encode(nullptr, 0, cdata)) {
+ return false;
+ }
+ }
+ // Note that a call to stream_encode will update cdata to apply the gzip
+ // header and that call itself will update _downstream_length. Since we don't
+ // want to double count the gzip header bytes, we capture initial_cdata_size
+ // here after any possible call to stream_encode above.
+ const auto initial_cdata_size = cdata.size();
char buf[BUF_SIZE];
_zstrm.zalloc = Z_NULL;
@@ -145,7 +156,7 @@ EsiGzip::stream_finish(std::string &cdata, int &downstream_length)
}
append(cdata, static_cast<uint32_t>(_crc));
append(cdata, static_cast<int32_t>(_total_data_length));
- _downstream_length += cdata.size();
+ _downstream_length += cdata.size() - initial_cdata_size;
downstream_length = _downstream_length;
return true;
}
diff --git a/plugins/esi/lib/EsiGzip.h b/plugins/esi/lib/EsiGzip.h
index 3fe9b1f..d6f6869 100644
--- a/plugins/esi/lib/EsiGzip.h
+++ b/plugins/esi/lib/EsiGzip.h
@@ -26,6 +26,7 @@
#include "ComponentBase.h"
#include <zlib.h>
#include <string>
+#include <string_view>
class EsiGzip : private EsiLib::ComponentBase
{
@@ -34,19 +35,41 @@ public:
~EsiGzip() override;
+ /** Compress the provided content.
+ *
+ * @param[in] data The input data to compress.
+ * @param[in] data_len The length of the input data to compress.
+ * @param[in,out] The result of compressing the input data will be appended
+ * to cdata.
+ *
+ * @return True if the compression succeeded, false otherwise.
+ */
bool stream_encode(const char *data, int data_len, std::string &cdata);
+ /** A string_view overload of stream_encode. */
inline bool
- stream_encode(std::string data, std::string &cdata)
+ stream_encode(std::string_view data, std::string &cdata)
{
return stream_encode(data.data(), data.size(), cdata);
}
+ /** Finish the compression stream.
+ *
+ * @param[out] cdata The compressed data is appended to this.
+ * @param[out] downstream_length The total number of compressed stream bytes
+ * across all calls to stream_encode and stream_finish.
+ *
+ * @return True if the compression succeeded, false otherwise.
+ */
bool stream_finish(std::string &cdata, int &downstream_length);
private:
- // int runDeflateLoop(z_stream &zstrm, int flush, std::string &cdata);
+ /** The cumulative total number of bytes for the compressed stream. */
int _downstream_length;
+
+ /** The cumulative total number of uncompressed bytes that have been
+ * compressed.
+ */
int _total_data_length;
z_stream _zstrm;
uLong _crc;
diff --git a/tests/gold_tests/pluginTest/esi/esi.test.py b/tests/gold_tests/pluginTest/esi/esi.test.py
new file mode 100644
index 0000000..a3b375c
--- /dev/null
+++ b/tests/gold_tests/pluginTest/esi/esi.test.py
@@ -0,0 +1,310 @@
+'''
+Test the ESI plugin.
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+Test.Summary = '''
+Test the ESI plugin.
+'''
+
+Test.SkipUnless(
+ Condition.PluginExists('esi.so'),
+)
+
+
+class EsiTest():
+ """
+ A class that encapsulates the configuration and execution of a set of EPI
+ test cases.
+ """
+
+ """ static: The same server Process is used across all tests. """
+ _server = None
+
+ """ static: A counter to keep the ATS process names unique across tests. """
+ _ts_counter = 0
+
+ """ static: A counter to keep any output file names unique across tests. """
+ _output_counter = 0
+
+ """ The ATS process for this set of test cases. """
+ _ts = None
+
+ def __init__(self, plugin_config):
+ """
+ Args:
+ plugin_config (str): The config line to place in plugin.config for
+ the ATS process.
+ """
+ if EsiTest._server is None:
+ EsiTest._server = EsiTest._create_server()
+
+ self._ts = EsiTest._create_ats(self, plugin_config)
+
+ @staticmethod
+ def _create_server():
+ """
+ Create and start a server process.
+ """
+ # Configure our server.
+ server = Test.MakeOriginServer("server")
+
+ # Generate the set of ESI responses derived right from our ESI docs.
+ # See:
+ # doc/admin-guide/plugins/esi.en.rst
+ request_header = {
+ "headers": (
+ "GET /esi.php HTTP/1.1\r\n"
+ "Host: www.example.com\r\n"
+ "Content-Length: 0\r\n\r\n"),
+ "timestamp": "1469733493.993",
+ "body": ""
+ }
+ esi_body = r'''<?php header('X-Esi: 1'); ?>
+<html>
+<body>
+Hello, <esi:include src="http://www.example.com/date.php"/>
+</body>
+</html>
+'''
+ response_header = {
+ "headers": (
+ "HTTP/1.1 200 OK\r\n"
+ "Content-Type: text/html\r\n"
+ "X-Esi: 1\r\n"
+ "Connection: close\r\n"
+ "Content-Length: {}\r\n"
+ "Cache-Control: max-age=300\r\n"
+ "\r\n".format(len(esi_body))),
+ "timestamp": "1469733493.993",
+ "body": esi_body
+ }
+ server.addResponse("sessionfile.log", request_header, response_header)
+ request_header = {
+ "headers": (
+ "GET /date.php HTTP/1.1\r\n"
+ "Host: www.example.com\r\n"
+ "Content-Length: 0\r\n\r\n"),
+ "timestamp": "1469733493.993",
+ "body": ""
+ }
+ date_body = r'''<?php
+header ("Cache-control: no-cache");
+echo date('l jS \of F Y h:i:s A');
+?>
+'''
+ response_header = {
+ "headers": (
+ "HTTP/1.1 200 OK\r\n"
+ "Content-Type: text/html\r\n"
+ "Connection: close\r\n"
+ "Content-Length: {}\r\n"
+ "Cache-Control: max-age=300\r\n"
+ "\r\n".format(len(date_body))),
+ "timestamp": "1469733493.993",
+ "body": date_body
+ }
+ server.addResponse("sessionfile.log", request_header, response_header)
+ # Verify correct functionality with an empty body.
+ request_header = {
+ "headers": (
+ "GET /expect_empty_body HTTP/1.1\r\n"
+ "Host: www.example.com\r\n"
+ "Content-Length: 0\r\n\r\n"),
+ "timestamp": "1469733493.993",
+ "body": ""
+ }
+ response_header = {
+ "headers": (
+ "HTTP/1.1 200 OK\r\n"
+ "X-ESI: On\r\n"
+ "Content-Length: 0\r\n"
+ "Connection: close\r\n"
+ "Content-Type: text/html; charset=UTF-8\r\n"
+ "\r\n"),
+ "timestamp": "1469733493.993",
+ "body": ""
+ }
+ server.addResponse("sessionfile.log", request_header, response_header)
+
+ # Create a run to start the server.
+ tr = Test.AddTestRun("Start the server.")
+ tr.Processes.Default.StartBefore(server)
+ tr.Processes.Default.Command = "echo starting the server"
+ tr.Processes.Default.ReturnCode = 0
+ tr.StillRunningAfter = server
+
+ return server
+
+ @staticmethod
+ def _create_ats(self, plugin_config):
+ """
+ Create and start an ATS process.
+ """
+ EsiTest._ts_counter += 1
+
+ # Configure ATS with a vanilla ESI plugin configuration.
+ ts = Test.MakeATSProcess("ts{}".format(EsiTest._ts_counter))
+ ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'http|plugin_esi',
+ })
+ ts.Disk.remap_config.AddLine(
+ 'map http://www.example.com/ http://127.0.0.1:{0}'.format(EsiTest._server.Variables.Port)
+ )
+ ts.Disk.plugin_config.AddLine(plugin_config)
+
+ # Create a run to start the ATS process.
+ tr = Test.AddTestRun("Start the ATS process.")
+ tr.Processes.Default.StartBefore(ts)
+ tr.Processes.Default.Command = "echo starting ATS"
+ tr.Processes.Default.ReturnCode = 0
+ tr.StillRunningAfter = ts
+ return ts
+
+ def run_cases_expecting_gzip(self):
+ # Test 1: Verify basic ESI functionality.
+ tr = Test.AddTestRun("First request for esi.php: not cached")
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/esi.php -H"Host: www.example.com" '
+ '-H"Accept: */*" --verbose'.format(
+ self._ts.Variables.port))
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.Streams.stderr = "gold/esi_headers.gold"
+ tr.Processes.Default.Streams.stdout = "gold/esi_body.gold"
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+
+ # Test 2: Repeat the above, should now be cached.
+ tr = Test.AddTestRun("Second request for esi.php: will be cached")
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/esi.php -H"Host: www.example.com" '
+ '-H"Accept: */*" --verbose'.format(
+ self._ts.Variables.port))
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.Streams.stderr = "gold/esi_headers.gold"
+ tr.Processes.Default.Streams.stdout = "gold/esi_body.gold"
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+
+ # Test 3: Verify the ESI plugin can gzip a response when the client accepts it.
+ tr = Test.AddTestRun("Verify the ESI plugin can gzip a response")
+ EsiTest._output_counter += 1
+ unzipped_body_file = os.path.join(
+ tr.RunDirectory,
+ "non_empty_curl_output_{}".format(EsiTest._output_counter))
+ gzipped_body_file = unzipped_body_file + ".gz"
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/esi.php -H"Host: www.example.com" '
+ '-H "Accept-Encoding: gzip" -H"Accept: */*" --verbose --output {1}'.format(
+ self._ts.Variables.port, gzipped_body_file))
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.Ready = When.FileExists(gzipped_body_file)
+ tr.Processes.Default.Streams.stderr = "gold/esi_gzipped.gold"
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+ zipped_body_disk_file = tr.Disk.File(gzipped_body_file)
+ zipped_body_disk_file.Exists = True
+
+ # Now, unzip the file and make sure its size is the expected body.
+ tr = Test.AddTestRun("Verify the file uzips to the expected body.")
+ tr.Processes.Default.Command = "gunzip {}".format(gzipped_body_file)
+ tr.Processes.Default.Ready = When.FileExists(unzipped_body_file)
+ tr.Processes.Default.ReturnCode = 0
+ unzipped_body_disk_file = tr.Disk.File(unzipped_body_file)
+ unzipped_body_disk_file.Content = "gold/esi_body.gold"
+
+ # Test 4: Verify correct handling of a gzipped empty response body.
+ tr = Test.AddTestRun("Verify we can handle an empty response.")
+ EsiTest._output_counter += 1
+ empty_body_file = os.path.join(
+ tr.RunDirectory,
+ "empty_curl_output_{}".format(EsiTest._output_counter))
+ gzipped_empty_body = empty_body_file + ".gz"
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/expect_empty_body -H"Host: www.example.com" '
+ '-H"Accept-Encoding: gzip" -H"Accept: */*" --verbose --output {1}'.format(
+ self._ts.Variables.port, gzipped_empty_body))
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.Ready = When.FileExists(gzipped_empty_body)
+ tr.Processes.Default.Streams.stderr = "gold/empty_response_body.gold"
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+ # The gzipped output file should be greater than 0, even though 0 bytes are
+ # compressed.
+ gz_disk_file = tr.Disk.File(gzipped_empty_body)
+ gz_disk_file.Size = Testers.GreaterThan(0)
+
+ # Now, unzip the file and make sure its size is the original 0 size body.
+ tr = Test.AddTestRun("Verify the file uzips to a zero sized file.")
+ tr.Processes.Default.Command = "gunzip {}".format(gzipped_empty_body)
+ tr.Processes.Default.Ready = When.FileExists(empty_body_file)
+ tr.Processes.Default.ReturnCode = 0
+ unzipped_disk_file = tr.Disk.File(empty_body_file)
+ unzipped_disk_file.Size = 0
+
+ def run_cases_expecting_no_gzip(self):
+ # Test 1: Run an ESI test where the client does not accept gzip.
+ tr = Test.AddTestRun("First request for esi.php: gzip not accepted.")
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/esi.php -H"Host: www.example.com" '
+ '-H"Accept: */*" --verbose'.format(
+ self._ts.Variables.port))
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.Streams.stderr = "gold/esi_headers.gold"
+ tr.Processes.Default.Streams.stdout = "gold/esi_body.gold"
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+
+ # Test 2: Verify the ESI plugin does not gzip the response even if the
+ # client accepts the gzip encoding.
+ tr = Test.AddTestRun("Verify the ESI plugin refuses to gzip responses with -–disable-gzip-output")
+ tr.Processes.Default.Command = \
+ ('curl http://127.0.0.1:{0}/esi.php -H"Host: www.example.com" '
+ '-H "Accept-Encoding: gzip" -H"Accept: */*" --verbose'.format(
+ self._ts.Variables.port))
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.Streams.stderr = "gold/esi_headers.gold"
+ tr.Processes.Default.Streams.stdout = "gold/esi_body.gold"
+ tr.StillRunningAfter = self._server
+ tr.StillRunningAfter = self._ts
+
+
+#
+# Configure and run the test cases.
+#
+
+# Run the tests with ESI configured with no parameters.
+vanilla_test = EsiTest(plugin_config='esi.so')
+vanilla_test.run_cases_expecting_gzip()
+
+# For these test cases, the behavior should remain the same with
+# --first-byte-flush set.
+first_byte_flush_test = EsiTest(plugin_config='esi.so --first-byte-flush')
+first_byte_flush_test.run_cases_expecting_gzip()
+
+# For these test cases, the behavior should remain the same with
+# --packed-node-support set.
+packed_node_support_test = EsiTest(plugin_config='esi.so --packed-node-support')
+packed_node_support_test.run_cases_expecting_gzip()
+
+# Run a set of cases verifying that the plugin does not zip content if
+# --disable-gzip-output is set.
+gzip_disabled_test = EsiTest(plugin_config='esi.so --disable-gzip-output')
+gzip_disabled_test.run_cases_expecting_no_gzip()
diff --git a/tests/gold_tests/pluginTest/esi/gold/empty_response_body.gold b/tests/gold_tests/pluginTest/esi/gold/empty_response_body.gold
new file mode 100644
index 0000000..994a61e
--- /dev/null
+++ b/tests/gold_tests/pluginTest/esi/gold/empty_response_body.gold
@@ -0,0 +1,14 @@
+``
+> GET /expect_empty_body HTTP/1.1
+> Host: www.example.com
+> User-Agent: ``
+> Accept-Encoding: gzip
+> Accept: */*
+``
+< HTTP/1.1 200 OK
+< Content-Type: text/html; charset=UTF-8
+``
+< Content-Length: 20
+``
+< Content-Encoding: gzip
+``
diff --git a/tests/gold_tests/pluginTest/esi/gold/esi_body.gold b/tests/gold_tests/pluginTest/esi/gold/esi_body.gold
new file mode 100644
index 0000000..a087e56
--- /dev/null
+++ b/tests/gold_tests/pluginTest/esi/gold/esi_body.gold
@@ -0,0 +1,12 @@
+``
+<?php header('X-Esi: 1'); ?>
+<html>
+<body>
+Hello, <?php
+header ("Cache-control: no-cache");
+echo date('l jS \of F Y h:i:s A');
+?>
+
+</body>
+</html>
+``
diff --git a/tests/gold_tests/pluginTest/esi/gold/esi_gzipped.gold b/tests/gold_tests/pluginTest/esi/gold/esi_gzipped.gold
new file mode 100644
index 0000000..025d13b
--- /dev/null
+++ b/tests/gold_tests/pluginTest/esi/gold/esi_gzipped.gold
@@ -0,0 +1,10 @@
+``
+> GET /esi.php HTTP/1.1
+``
+> Accept-Encoding: gzip
+> Accept: */*
+``
+< HTTP/1.1 200 OK
+``
+< Content-Encoding: gzip
+``
diff --git a/tests/gold_tests/pluginTest/esi/gold/esi_headers.gold b/tests/gold_tests/pluginTest/esi/gold/esi_headers.gold
new file mode 100644
index 0000000..4e9d577
--- /dev/null
+++ b/tests/gold_tests/pluginTest/esi/gold/esi_headers.gold
@@ -0,0 +1,6 @@
+``
+> GET /esi.php HTTP/1.1
+``
+< HTTP/1.1 200 OK
+< Content-Type: text/html
+``