You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2020/06/09 21:49:59 UTC
[trafficserver] branch 9.0.x updated: slice plugin: add
--include-regex, --exclude-regex parameters (#6701)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/9.0.x by this push:
new 1663521 slice plugin: add --include-regex, --exclude-regex parameters (#6701)
1663521 is described below
commit 166352124806c9753b3efe82e2f4ac3ae41f9e91
Author: Brian Olsen <br...@comcast.com>
AuthorDate: Tue Apr 28 13:30:54 2020 -0600
slice plugin: add --include-regex, --exclude-regex parameters (#6701)
(cherry picked from commit 8fdf694436e60131a5d8a9e2728e8c5aa0fe167a)
---
doc/admin-guide/plugins/slice.en.rst | 21 +++
plugins/experimental/slice/Config.cc | 121 ++++++++++++---
plugins/experimental/slice/Config.h | 28 +++-
plugins/experimental/slice/HttpHeader.cc | 12 +-
plugins/experimental/slice/Makefile.inc | 6 +
plugins/experimental/slice/slice.cc | 22 ++-
.../pluginTest/slice/slice_regex.test.py | 169 +++++++++++++++++++++
7 files changed, 346 insertions(+), 33 deletions(-)
diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst
index f035e82..70bd255 100644
--- a/doc/admin-guide/plugins/slice.en.rst
+++ b/doc/admin-guide/plugins/slice.en.rst
@@ -88,6 +88,17 @@ The slice plugin supports the following options::
Disable writing block stitch errors to the error log.
-d for short
+ --exclude-regex=<regex> (optional)
+ If provided, only slice what matches.
+ If not provided will always slice
+ Cannot be used with --include-regex
+ -e for short
+
+ --include-regex=<regex> (optional)
+ If provided, only slice what matches.
+ If not provided will always slice
+ Cannot be used with --exclude-regex
+ -i for short
Examples::
@@ -122,6 +133,16 @@ After modifying :file:`remap.config`, restart or reload traffic server
(sudo traffic_ctl config reload) or (sudo traffic_ctl server restart)
to activate the new configuration values.
+Don't slice txt files::
+
+ slice.so --exclude-regex=\\.txt
+ slice.so -e \\.txt
+
+Slice only mp4 files::
+
+ slice.so --include-regex=\\.mp4
+ slice.so -i \\.mp4
+
Debug Options
-------------
diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc
index fb04ebb..17b7329 100644
--- a/plugins/experimental/slice/Config.cc
+++ b/plugins/experimental/slice/Config.cc
@@ -26,6 +26,20 @@
#include "ts/experimental.h"
+Config::~Config()
+{
+ if (nullptr != m_regex_extra) {
+#ifndef PCRE_STUDY_JIT_COMPILE
+ pcre_free(m_regex_extra);
+#else
+ pcre_free_study(m_regex_extra);
+#endif
+ }
+ if (nullptr != m_regex) {
+ pcre_free(m_regex);
+ }
+}
+
int64_t
Config::bytesFrom(char const *const valstr)
{
@@ -94,18 +108,20 @@ Config::fromArgs(int const argc, char const *const argv[])
// standard parsing
constexpr struct option longopts[] = {
{const_cast<char *>("blockbytes"), required_argument, nullptr, 'b'},
- {const_cast<char *>("blockbytes-test"), required_argument, nullptr, 't'},
- {const_cast<char *>("remap-host"), required_argument, nullptr, 'r'},
- {const_cast<char *>("pace-errorlog"), required_argument, nullptr, 'p'},
{const_cast<char *>("disable-errorlog"), no_argument, nullptr, 'd'},
+ {const_cast<char *>("exclude-regex"), required_argument, nullptr, 'e'},
+ {const_cast<char *>("include-regex"), required_argument, nullptr, 'i'},
{const_cast<char *>("throttle"), no_argument, nullptr, 'o'},
+ {const_cast<char *>("pace-errorlog"), required_argument, nullptr, 'p'},
+ {const_cast<char *>("remap-host"), required_argument, nullptr, 'r'},
+ {const_cast<char *>("blockbytes-test"), required_argument, nullptr, 't'},
{nullptr, 0, nullptr, 0},
};
// getopt assumes args start at '1' so this hack is needed
char *const *argvp = (const_cast<char *const *>(argv) - 1);
for (;;) {
- int const opt = getopt_long(argc + 1, argvp, "b:t:r:p:do", longopts, nullptr);
+ int const opt = getopt_long(argc + 1, argvp, "b:de:i:op:r:t:", longopts, nullptr);
if (-1 == opt) {
break;
}
@@ -122,6 +138,61 @@ Config::fromArgs(int const argc, char const *const argv[])
ERROR_LOG("Invalid blockbytes: %s", optarg);
}
} break;
+ case 'd': {
+ m_paceerrsecs = -1;
+ } break;
+ case 'e': {
+ if (None != m_regex_type) {
+ ERROR_LOG("Regex already specified!");
+ break;
+ }
+
+ const char *errptr;
+ int erroffset;
+ m_regexstr = optarg;
+ m_regex = pcre_compile(m_regexstr.c_str(), 0, &errptr, &erroffset, NULL);
+ if (nullptr == m_regex) {
+ ERROR_LOG("Invalid regex: '%s'", m_regexstr.c_str());
+ } else {
+ m_regex_type = Exclude;
+ m_regex_extra = pcre_study(m_regex, 0, &errptr);
+ DEBUG_LOG("Using regex for url exclude: '%s'", m_regexstr.c_str());
+ }
+ } break;
+ case 'i': {
+ if (None != m_regex_type) {
+ ERROR_LOG("Regex already specified!");
+ break;
+ }
+
+ const char *errptr;
+ int erroffset;
+ m_regexstr = optarg;
+ m_regex = pcre_compile(m_regexstr.c_str(), 0, &errptr, &erroffset, NULL);
+ if (nullptr == m_regex) {
+ ERROR_LOG("Invalid regex: '%s'", m_regexstr.c_str());
+ } else {
+ m_regex_type = Include;
+ m_regex_extra = pcre_study(m_regex, 0, &errptr);
+ DEBUG_LOG("Using regex for url include: '%s'", m_regexstr.c_str());
+ }
+ } break;
+ case 'o': {
+ m_throttle = true;
+ DEBUG_LOG("Enabling internal block throttling");
+ } break;
+ case 'p': {
+ int const secsread = atoi(optarg);
+ if (0 < secsread) {
+ m_paceerrsecs = std::min(secsread, 60);
+ } else {
+ ERROR_LOG("Ignoring pace-errlog argument");
+ }
+ } break;
+ case 'r': {
+ m_remaphost = optarg;
+ DEBUG_LOG("Using loopback remap host override: %s", m_remaphost.c_str());
+ } break;
case 't': {
if (0 == blockbytes) {
int64_t const bytesread = bytesFrom(optarg);
@@ -135,25 +206,6 @@ Config::fromArgs(int const argc, char const *const argv[])
DEBUG_LOG("Skipping blockbytes-test in favor of blockbytes");
}
} break;
- case 'r':
- m_remaphost = optarg;
- DEBUG_LOG("Using loopback remap host override: %s", m_remaphost.c_str());
- break;
- case 'p': {
- int const secsread = atoi(optarg);
- if (0 < secsread) {
- m_paceerrsecs = std::min(secsread, 60);
- } else {
- ERROR_LOG("Ignoring pace-errlog argument");
- }
- } break;
- case 'd':
- m_paceerrsecs = -1;
- break;
- case 'o':
- m_throttle = true;
- DEBUG_LOG("Enabling internal block throttling");
- break;
default:
break;
}
@@ -204,3 +256,26 @@ Config::canLogError()
return true;
}
+
+bool
+Config::matchesRegex(char const *const url, int const urllen) const
+{
+ bool matches = true;
+
+ switch (m_regex_type) {
+ case Exclude: {
+ if (0 <= pcre_exec(m_regex, m_regex_extra, url, urllen, 0, 0, NULL, 0)) {
+ matches = false;
+ }
+ } break;
+ case Include: {
+ if (pcre_exec(m_regex, m_regex_extra, url, urllen, 0, 0, NULL, 0) < 0) {
+ matches = false;
+ }
+ } break;
+ default:
+ break;
+ }
+
+ return matches;
+}
diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h
index 42de33d..5edf463 100644
--- a/plugins/experimental/slice/Config.h
+++ b/plugins/experimental/slice/Config.h
@@ -20,6 +20,12 @@
#include "slice.h"
+#ifdef HAVE_PCRE_PCRE_H
+#include <pcre/pcre.h>
+#else
+#include <pcre.h>
+#endif
+
#include <mutex>
// Data Structures and Classes
@@ -30,18 +36,36 @@ struct Config {
int64_t m_blockbytes{blockbytesdefault};
std::string m_remaphost; // remap host to use for loopback slice GET
- bool m_throttle{false}; // internal block throttling
- int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s
+ std::string m_regexstr; // regex string for things to slice (default all)
+ enum RegexType { None, Include, Exclude };
+ RegexType m_regex_type{None};
+ pcre *m_regex{nullptr};
+ pcre_extra *m_regex_extra{nullptr};
+ bool m_throttle{false}; // internal block throttling
+ int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s
// Convert optarg to bytes
static int64_t bytesFrom(char const *const valstr);
+ // clean up pcre if applicable
+ ~Config();
+
// Parse from args, ast one wins
bool fromArgs(int const argc, char const *const argv[]);
// Check if the error should can be logged, if sucessful may update m_nexttime
bool canLogError();
+ // Check if regex supplied
+ bool
+ hasRegex() const
+ {
+ return None != m_regex_type;
+ }
+
+ // If no null reg, true, otherwise check against regex
+ bool matchesRegex(char const *const url, int const urllen) const;
+
private:
TSHRTime m_nextlogtime{0}; // next time to log in ns
std::mutex m_mutex;
diff --git a/plugins/experimental/slice/HttpHeader.cc b/plugins/experimental/slice/HttpHeader.cc
index 9913701..f70c42e 100644
--- a/plugins/experimental/slice/HttpHeader.cc
+++ b/plugins/experimental/slice/HttpHeader.cc
@@ -54,18 +54,20 @@ HttpHeader::setStatus(TSHttpStatus const newstatus)
}
char *
-HttpHeader ::urlString(int *const urllen) const
+HttpHeader::urlString(int *const urllen) const
{
char *urlstr = nullptr;
TSAssert(nullptr != urllen);
TSMLoc locurl = nullptr;
TSReturnCode const rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurl);
- if (TS_SUCCESS == rcode && nullptr != locurl) {
- urlstr = TSUrlStringGet(m_buffer, locurl, urllen);
+ if (nullptr != locurl) {
+ if (TS_SUCCESS == rcode) {
+ urlstr = TSUrlStringGet(m_buffer, locurl, urllen);
+ } else {
+ *urllen = 0;
+ }
TSHandleMLocRelease(m_buffer, m_lochdr, locurl);
- } else {
- *urllen = 0;
}
return urlstr;
diff --git a/plugins/experimental/slice/Makefile.inc b/plugins/experimental/slice/Makefile.inc
index 21f6166..be376ca 100644
--- a/plugins/experimental/slice/Makefile.inc
+++ b/plugins/experimental/slice/Makefile.inc
@@ -49,6 +49,8 @@ experimental_slice_test_content_range_SOURCES = \
experimental/slice/unit-tests/test_content_range.cc \
experimental/slice/ContentRange.cc
+experimental_slice_test_content_range_LDADD = @LIBPCRE@
+
check_PROGRAMS += experimental/slice/test_range
experimental_slice_test_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST
@@ -56,9 +58,13 @@ experimental_slice_test_range_SOURCES = \
experimental/slice/unit-tests/test_range.cc \
experimental/slice/Range.cc
+experimental_slice_test_range_LDADD = @LIBPCRE@
+
check_PROGRAMS += experimental/slice/test_config
experimental_slice_test_config_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST
experimental_slice_test_config_SOURCES = \
experimental/slice/unit-tests/test_config.cc \
experimental/slice/Config.cc
+
+experimental_slice_test_config_LDADD = @LIBPCRE@
diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc
index 0109f20..f53fdd3 100644
--- a/plugins/experimental/slice/slice.cc
+++ b/plugins/experimental/slice/slice.cc
@@ -59,10 +59,26 @@ read_request(TSHttpTxn txnp, Config *const config)
// check if any previous plugin has monkeyed with the transaction status
TSHttpStatus const txnstat = TSHttpTxnStatusGet(txnp);
if (0 != (int)txnstat) {
- DEBUG_LOG("slice: txn status change detected (%d), skipping plugin\n", (int)txnstat);
+ DEBUG_LOG("txn status change detected (%d), skipping plugin\n", (int)txnstat);
return false;
}
+ if (config->hasRegex()) {
+ int urllen = 0;
+ char *const urlstr = TSHttpTxnEffectiveUrlStringGet(txnp, &urllen);
+ if (nullptr != urlstr) {
+ bool const shouldslice = config->matchesRegex(urlstr, urllen);
+ if (!shouldslice) {
+ DEBUG_LOG("request failed regex, not slicing: '%.*s'", urllen, urlstr);
+ TSfree(urlstr);
+ return false;
+ }
+
+ DEBUG_LOG("request passed regex, slicing: '%.*s'", urllen, urlstr);
+ TSfree(urlstr);
+ }
+ }
+
// turn off any and all transaction caching (shouldn't matter)
TSHttpTxnServerRespNoStoreSet(txnp, 1);
TSHttpTxnRespCacheableSet(txnp, 0);
@@ -99,8 +115,8 @@ read_request(TSHttpTxn txnp, Config *const config)
// is the plugin configured to use a remap host?
std::string const &newhost = config->m_remaphost;
if (newhost.empty()) {
- TSMBuffer urlbuf;
- TSMLoc urlloc;
+ TSMBuffer urlbuf = nullptr;
+ TSMLoc urlloc = nullptr;
TSReturnCode rcode = TSHttpTxnPristineUrlGet(txnp, &urlbuf, &urlloc);
if (TS_SUCCESS == rcode) {
diff --git a/tests/gold_tests/pluginTest/slice/slice_regex.test.py b/tests/gold_tests/pluginTest/slice/slice_regex.test.py
new file mode 100644
index 0000000..f1a14b5
--- /dev/null
+++ b/tests/gold_tests/pluginTest/slice/slice_regex.test.py
@@ -0,0 +1,169 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Test.Summary = '''
+slice regex plugin test
+'''
+
+## Test description:
+# Preload the cache with the entire asset to be range requested.
+# Reload remap rule with slice plugin
+# Request content through the slice plugin
+
+Test.SkipUnless(
+ Condition.PluginExists('slice.so'),
+)
+Test.ContinueOnFail = False
+
+# configure origin server
+server = Test.MakeOriginServer("server")
+
+# Define ATS and configure
+ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True)
+
+# default root
+request_header_chk = {"headers":
+ "GET / HTTP/1.1\r\n" +
+ "Host: www.example.com\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+response_header_chk = {"headers":
+ "HTTP/1.1 200 OK\r\n" +
+ "Connection: close\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+server.addResponse("sessionlog.json", request_header_chk, response_header_chk)
+
+#block_bytes = 7
+body = "lets go surfin now"
+
+request_header_txt = {"headers":
+ "GET /slice.txt HTTP/1.1\r\n" +
+ "Host: slice\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+response_header_txt = {"headers":
+ "HTTP/1.1 200 OK\r\n" +
+ "Connection: close\r\n" +
+ 'Etag: "path"\r\n' +
+ "Cache-Control: max-age=500\r\n" +
+ "X-Info: notsliced\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body,
+}
+
+server.addResponse("sessionlog.json", request_header_txt, response_header_txt)
+
+request_header_mp4 = {"headers":
+ "GET /slice.mp4 HTTP/1.1\r\n" +
+ "Host: sliced\r\n" +
+ "Range: bytes=0-99\r\n"
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": "",
+}
+
+response_header_mp4 = {"headers":
+ "HTTP/1.1 206 Partial Content\r\n" +
+ "Connection: close\r\n" +
+ 'Etag: "path"\r\n' +
+ "Content-Range: bytes 0-{}/{}\r\n".format(len(body) - 1, len(body)) +
+ "Cache-Control: max-age=500\r\n" +
+ "X-Info: sliced\r\n" +
+ "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body,
+}
+
+server.addResponse("sessionlog.json", request_header_mp4, response_header_mp4)
+
+curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port)
+
+block_bytes = 100
+
+# set up whole asset fetch into cache
+ts.Disk.remap_config.AddLines([
+ 'map http://exclude/ http://127.0.0.1:{}/'.format(server.Variables.Port) +
+ ' @plugin=slice.so' +
+ ' @pparam=--blockbytes-test={}'.format(block_bytes) +
+ ' @pparam=--exclude-regex=\\.txt'
+ ' @pparam=--remap-host=sliced',
+ 'map http://include/ http://127.0.0.1:{}/'.format(server.Variables.Port) +
+ ' @plugin=slice.so' +
+ ' @pparam=--blockbytes-test={}'.format(block_bytes) +
+ ' @pparam=--include-regex=\\.mp4'
+ ' @pparam=--remap-host=sliced',
+ 'map http://sliced/ http://127.0.0.1:{}/'.format(server.Variables.Port),
+])
+
+
+# minimal configuration
+ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'slice',
+ 'proxy.config.http.cache.http': 0,
+ 'proxy.config.http.wait_for_cache': 0,
+ 'proxy.config.http.insert_age_in_response': 0,
+ 'proxy.config.http.response_via_str': 0,
+})
+
+# 0 Test - Exclude: ensure txt passes through
+tr = Test.AddTestRun("Exclude - asset passed through")
+ps = tr.Processes.Default
+ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
+ps.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port))
+ps.Command = curl_and_args + ' http://exclude/slice.txt'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: notsliced", "expected not sliced header")
+tr.StillRunningAfter = ts
+
+# 1 Test - Exclude mp4 gets sliced
+tr = Test.AddTestRun("Exclude - asset is sliced")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://exclude/slice.mp4'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: sliced", "expected sliced header")
+tr.StillRunningAfter = ts
+tr.StillRunningAfter = ts
+
+# 2 Test - Exclude: ensure txt passes through
+tr = Test.AddTestRun("Include - asset passed through")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://include/slice.txt'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: notsliced", "expected not sliced header")
+tr.StillRunningAfter = ts
+
+# 3 Test - Exclude mp4 gets sliced
+tr = Test.AddTestRun("Include - asset is sliced")
+ps = tr.Processes.Default
+ps.Command = curl_and_args + ' http://include/slice.mp4'
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: sliced", "expected sliced header")
+tr.StillRunningAfter = ts
+tr.StillRunningAfter = ts