You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by bc...@apache.org on 2018/03/08 21:18:07 UTC
[trafficserver] branch master updated (3e00ae3 -> 88c74e4)
This is an automated email from the ASF dual-hosted git repository.
bcall pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git.
from 3e00ae3 docs for client read error
new 78cb6c9 TS-4042: Add feature to buffer request body before making downstream requests
new 88c74e4 Adding slow_post_test
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
example/Makefile.am | 2 +
example/request_buffer/request_buffer.c | 148 +++++++++++++++++++++
lib/ts/apidefs.h.in | 3 +
plugins/experimental/ts_lua/ts_lua_http_config.c | 2 +
proxy/InkAPI.cc | 13 ++
proxy/InkAPITest.cc | 6 +-
proxy/api/ts/ts.h | 5 +
proxy/http/HttpConfig.cc | 3 +
proxy/http/HttpConfig.h | 6 +
proxy/http/HttpDebugNames.cc | 6 +
proxy/http/HttpSM.cc | 95 ++++++++++++-
proxy/http/HttpSM.h | 48 +++++++
proxy/http/HttpTransact.cc | 130 +++++++++---------
proxy/http/HttpTransact.h | 3 +
proxy/http/HttpTunnel.cc | 33 +++--
proxy/http/HttpTunnel.h | 9 +-
tests/gold_tests/slow_post/gold/200.gold | 1 +
tests/gold_tests/slow_post/slow_post.test.py | 73 ++++++++++
.../slow_post_client.py} | 44 +++---
tests/tools/plugins/request_buffer.c | 144 ++++++++++++++++++++
20 files changed, 665 insertions(+), 109 deletions(-)
create mode 100644 example/request_buffer/request_buffer.c
create mode 100644 tests/gold_tests/slow_post/gold/200.gold
create mode 100644 tests/gold_tests/slow_post/slow_post.test.py
copy tests/gold_tests/{h2/h2active_timeout.py => slow_post/slow_post_client.py} (59%)
create mode 100644 tests/tools/plugins/request_buffer.c
--
To stop receiving notification emails like this one, please contact
bcall@apache.org.
[trafficserver] 01/02: TS-4042: Add feature to buffer request body
before making downstream requests
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bcall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 78cb6c9bf86e8d72c79a9084604bc25520ef57d7
Author: Zizhong Zhang <zi...@linkedin.com>
AuthorDate: Fri Aug 4 16:27:48 2017 -0700
TS-4042: Add feature to buffer request body before making downstream requests
---
example/Makefile.am | 2 +
example/request_buffer/request_buffer.c | 148 +++++++++++++++++++++++
lib/ts/apidefs.h.in | 3 +
plugins/experimental/ts_lua/ts_lua_http_config.c | 2 +
proxy/InkAPI.cc | 13 ++
proxy/InkAPITest.cc | 6 +-
proxy/api/ts/ts.h | 5 +
proxy/http/HttpConfig.cc | 3 +
proxy/http/HttpConfig.h | 6 +
proxy/http/HttpDebugNames.cc | 6 +
proxy/http/HttpSM.cc | 95 ++++++++++++++-
proxy/http/HttpSM.h | 48 ++++++++
proxy/http/HttpTransact.cc | 130 +++++++++++---------
proxy/http/HttpTransact.h | 3 +
proxy/http/HttpTunnel.cc | 33 +++--
proxy/http/HttpTunnel.h | 9 +-
16 files changed, 424 insertions(+), 88 deletions(-)
diff --git a/example/Makefile.am b/example/Makefile.am
index c79c3a3..6c816ee 100644
--- a/example/Makefile.am
+++ b/example/Makefile.am
@@ -32,6 +32,7 @@ example_Plugins = \
blacklist_0.la \
blacklist_1.la \
bnull_transform.la \
+ request_buffer.la \
cache_scan.la \
file_1.la \
hello.la \
@@ -99,6 +100,7 @@ basic_auth_la_SOURCES = basic_auth/basic_auth.c
blacklist_0_la_SOURCES = blacklist_0/blacklist_0.c
blacklist_1_la_SOURCES = blacklist_1/blacklist_1.c
bnull_transform_la_SOURCES = bnull_transform/bnull_transform.c
+request_buffer_la_SOURCES = request_buffer/request_buffer.c
cache_scan_la_SOURCES = cache_scan/cache_scan.cc
file_1_la_SOURCES = file_1/file_1.c
hello_la_SOURCES = hello/hello.c
diff --git a/example/request_buffer/request_buffer.c b/example/request_buffer/request_buffer.c
new file mode 100644
index 0000000..a75650f
--- /dev/null
+++ b/example/request_buffer/request_buffer.c
@@ -0,0 +1,148 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ts/ts.h"
+#include "ts/ink_assert.h"
+#include "ts/ink_defs.h"
+
+#define PLUGIN_NAME "request_buffer"
+
+#define TS_NULL_MUTEX NULL
+
+static char *
+request_body_get(TSHttpTxn txnp, int *len)
+{
+ char *ret = NULL;
+ TSIOBufferReader post_buffer_reader = TSHttpTxnPostBufferReaderGet(txnp);
+ int64_t read_avail = TSIOBufferReaderAvail(post_buffer_reader);
+ if (read_avail == 0) {
+ TSIOBufferReaderFree(post_buffer_reader);
+ return NULL;
+ }
+
+ ret = (char *)TSmalloc(sizeof(char) * read_avail);
+
+ int64_t consumed = 0;
+ int64_t data_len = 0;
+ const char *char_data = NULL;
+ TSIOBufferBlock block = TSIOBufferReaderStart(post_buffer_reader);
+ while (block != NULL) {
+ char_data = TSIOBufferBlockReadStart(block, post_buffer_reader, &data_len);
+ memcpy(ret + consumed, char_data, data_len);
+ consumed += data_len;
+ block = TSIOBufferBlockNext(block);
+ }
+ TSIOBufferReaderFree(post_buffer_reader);
+
+ *len = (int)consumed;
+ return ret;
+}
+
+static int
+request_buffer_plugin(TSCont contp, TSEvent event, void *edata)
+{
+ TSDebug(PLUGIN_NAME, "request_buffer_plugin starting, event[%d]", event);
+ TSHttpTxn txnp = (TSHttpTxn)(edata);
+ if (event == TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE) {
+ int len = 0;
+ char *body = request_body_get(txnp, &len);
+ TSDebug(PLUGIN_NAME, "request_buffer_plugin gets the request body with length[%d]", len);
+ TSfree(body);
+ TSContDestroy(contp);
+ } else {
+ ink_assert(0);
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+}
+
+bool
+is_post_request(TSHttpTxn txnp)
+{
+ TSMLoc req_loc;
+ TSMBuffer req_bufp;
+ if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_loc) == TS_ERROR) {
+ TSError("Error while retrieving client request header\n");
+ return false;
+ }
+ int method_len = 0;
+ const char *method = TSHttpHdrMethodGet(req_bufp, req_loc, &method_len);
+ if (method_len != (int)strlen(TS_HTTP_METHOD_POST) || strncasecmp(method, TS_HTTP_METHOD_POST, method_len) != 0) {
+ TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc);
+ return false;
+ }
+ TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc);
+ return true;
+}
+
+static int
+global_plugin(TSCont contp ATS_UNUSED, TSEvent event, void *edata)
+{
+ TSDebug(PLUGIN_NAME, "transform_plugin starting");
+ TSHttpTxn txnp = (TSHttpTxn)edata;
+
+ switch (event) {
+ case TS_EVENT_HTTP_READ_REQUEST_HDR:
+ if (is_post_request(txnp)) {
+ TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, 1);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TSContCreate(request_buffer_plugin, TSMutexCreate()));
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+void
+TSPluginInit(int argc ATS_UNUSED, const char *argv[] ATS_UNUSED)
+{
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = PLUGIN_NAME;
+ info.vendor_name = "Apache Software Foundation";
+ info.support_email = "dev@trafficserver.apache.org";
+
+ if (TSPluginRegister(&info) != TS_SUCCESS) {
+ TSDebug(PLUGIN_NAME, "[%s] Plugin registration failed", PLUGIN_NAME);
+
+ goto Lerror;
+ }
+
+ /* This is call we could use if we need to protect global data */
+ /* TSReleaseAssert ((mutex = TSMutexCreate()) != TS_NULL_MUTEX); */
+
+ TSMutex mutex = TS_NULL_MUTEX;
+ TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(global_plugin, mutex));
+ TSDebug(PLUGIN_NAME, "[%s] Plugin registration succeeded", PLUGIN_NAME);
+ return;
+
+Lerror:
+ TSDebug(PLUGIN_NAME, "[%s] Plugin disabled", PLUGIN_NAME);
+}
diff --git a/lib/ts/apidefs.h.in b/lib/ts/apidefs.h.in
index 0889181..a035812 100644
--- a/lib/ts/apidefs.h.in
+++ b/lib/ts/apidefs.h.in
@@ -294,6 +294,7 @@ typedef enum {
TS_SSL_VERIFY_CLIENT_HOOK,
TS_SSL_SESSION_HOOK,
TS_SSL_LAST_HOOK = TS_SSL_SESSION_HOOK,
+ TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK = 23,
TS_HTTP_LAST_HOOK
} TSHttpHookID;
@@ -452,6 +453,7 @@ typedef enum {
TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60022,
TS_EVENT_VCONN_PRE_ACCEPT = 60023,
TS_EVENT_LIFECYCLE_MSG = 60024,
+ TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60025,
TS_EVENT_MGMT_UPDATE = 60100,
TS_EVENT_INTERNAL_60200 = 60200,
TS_EVENT_INTERNAL_60201 = 60201,
@@ -766,6 +768,7 @@ typedef enum {
TS_CONFIG_HTTP_NORMALIZE_AE,
TS_CONFIG_HTTP_INSERT_FORWARDED,
TS_CONFIG_HTTP_ALLOW_MULTI_RANGE,
+ TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED,
TS_CONFIG_LAST_ENTRY
} TSOverridableConfigKey;
diff --git a/plugins/experimental/ts_lua/ts_lua_http_config.c b/plugins/experimental/ts_lua/ts_lua_http_config.c
index 8bf78c7..c4b7593 100644
--- a/plugins/experimental/ts_lua/ts_lua_http_config.c
+++ b/plugins/experimental/ts_lua/ts_lua_http_config.c
@@ -133,6 +133,7 @@ typedef enum {
TS_LUA_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS = TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS,
TS_LUA_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT = TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT,
TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE = TS_CONFIG_HTTP_ALLOW_MULTI_RANGE,
+ TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED = TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED,
TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY,
} TSLuaOverridableConfigKey;
@@ -258,6 +259,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS),
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT),
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE),
+ TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED),
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY),
};
diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc
index 5e35f89..5dd1319 100644
--- a/proxy/InkAPI.cc
+++ b/proxy/InkAPI.cc
@@ -8115,6 +8115,9 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr
case TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED:
ret = _memberp_to_generic(&overridableHttpConfig->post_check_content_length_enabled, typep);
break;
+ case TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED:
+ ret = _memberp_to_generic(&overridableHttpConfig->request_buffer_enabled, typep);
+ break;
case TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER:
ret = _memberp_to_generic(&overridableHttpConfig->global_user_agent_header, typep);
break;
@@ -8595,6 +8598,8 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf,
case 'd':
if (!strncmp(name, "proxy.config.http.forward_connect_method", length)) {
cnf = TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD;
+ } else if (!strncmp(name, "proxy.config.http.request_buffer_enabled", length)) {
+ cnf = TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED;
}
break;
case 'e':
@@ -9630,3 +9635,11 @@ TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp)
{
return remapUrlGet(txnp, urlLocp, &UrlMappingContainer::getToURL);
}
+
+tsapi TSIOBufferReader
+TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp)
+{
+ sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS);
+ HttpSM *sm = (HttpSM *)txnp;
+ return (TSIOBufferReader)sm->get_postbuf_clone_reader();
+}
diff --git a/proxy/InkAPITest.cc b/proxy/InkAPITest.cc
index 12b6aae..8bcdae8 100644
--- a/proxy/InkAPITest.cc
+++ b/proxy/InkAPITest.cc
@@ -5546,7 +5546,8 @@ typedef enum {
ORIG_TS_SSL_SERVER_VERIFY_HOOK,
ORIG_TS_SSL_VERIFY_CLIENT_HOOK,
ORIG_TS_SSL_SESSION_HOOK,
- ORIG_TS_SSL_LAST_HOOK = ORIG_TS_SSL_SESSION_HOOK,
+ ORIG_TS_SSL_LAST_HOOK = ORIG_TS_SSL_SESSION_HOOK,
+ ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK = 23,
ORIG_TS_HTTP_LAST_HOOK
} ORIG_TSHttpHookID;
@@ -7604,7 +7605,8 @@ const char *SDK_Overridable_Configs[TS_CONFIG_LAST_ENTRY] = {"proxy.config.url_r
"proxy.config.http.parent_proxy.connect_attempts_timeout",
"proxy.config.http.normalize_ae",
"proxy.config.http.insert_forwarded",
- "proxy.config.http.allow_multi_range"};
+ "proxy.config.http.allow_multi_range",
+ "proxy.config.http.request_buffer_enabled"};
REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus)
{
diff --git a/proxy/api/ts/ts.h b/proxy/api/ts/ts.h
index 7b1b752..2f07c64 100644
--- a/proxy/api/ts/ts.h
+++ b/proxy/api/ts/ts.h
@@ -2460,6 +2460,11 @@ tsapi TSReturnCode TSRemapFromUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp);
//
tsapi TSReturnCode TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp);
+/*
+ * Get a TSIOBufferReader to read the buffered body. The return value needs to be freed.
+ */
+tsapi TSIOBufferReader TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp);
+
#ifdef __cplusplus
}
#endif /* __cplusplus */
diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc
index 6f08ead..65990ef 100644
--- a/proxy/http/HttpConfig.cc
+++ b/proxy/http/HttpConfig.cc
@@ -966,6 +966,7 @@ HttpConfig::startup()
HttpEstablishStaticConfigLongLong(c.oride.flow_high_water_mark, "proxy.config.http.flow_control.high_water");
HttpEstablishStaticConfigLongLong(c.oride.flow_low_water_mark, "proxy.config.http.flow_control.low_water");
HttpEstablishStaticConfigByte(c.oride.post_check_content_length_enabled, "proxy.config.http.post.check.content_length.enabled");
+ HttpEstablishStaticConfigByte(c.oride.request_buffer_enabled, "proxy.config.http.request_buffer_enabled");
HttpEstablishStaticConfigByte(c.strict_uri_parsing, "proxy.config.http.strict_uri_parsing");
// [amc] This is a bit of a mess, need to figure out to make this cleaner.
@@ -1247,6 +1248,8 @@ HttpConfig::reconfigure()
params->oride.post_check_content_length_enabled = INT_TO_BOOL(m_master.oride.post_check_content_length_enabled);
+ params->oride.request_buffer_enabled = INT_TO_BOOL(m_master.oride.request_buffer_enabled);
+
params->oride.flow_control_enabled = INT_TO_BOOL(m_master.oride.flow_control_enabled);
params->oride.flow_high_water_mark = m_master.oride.flow_high_water_mark;
params->oride.flow_low_water_mark = m_master.oride.flow_low_water_mark;
diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h
index a01a2d9..8a22a43 100644
--- a/proxy/http/HttpConfig.h
+++ b/proxy/http/HttpConfig.h
@@ -450,6 +450,7 @@ struct OverridableHttpConfigParams {
parent_failures_update_hostdb(0),
cache_open_write_fail_action(0),
post_check_content_length_enabled(1),
+ request_buffer_enabled(0),
ssl_client_verify_server(0),
redirect_use_orig_cache_key(0),
number_of_redirections(0),
@@ -624,6 +625,11 @@ struct OverridableHttpConfigParams {
////////////////////////
MgmtByte post_check_content_length_enabled;
+ ////////////////////////////////////////////////
+ // Buffer post body before connecting servers //
+ ////////////////////////////////////////////////
+ MgmtByte request_buffer_enabled;
+
/////////////////////////////
// server verification mode//
/////////////////////////////
diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc
index e135eef..048febd 100644
--- a/proxy/http/HttpDebugNames.cc
+++ b/proxy/http/HttpDebugNames.cc
@@ -352,6 +352,10 @@ HttpDebugNames::get_action_name(HttpTransact::StateMachineAction_t e)
return ("SM_ACTION_DRAIN_REQUEST_BODY");
#endif /* PROXY_DRAIN */
+ case HttpTransact::SM_ACTION_WAIT_FOR_FULL_BODY:
+ return ("SM_ACTION_WAIT_FOR_FULL_BODY");
+ case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE:
+ return ("SM_ACTION_REQUEST_BUFFER_READ_COMPLETE");
case HttpTransact::SM_ACTION_API_SM_START:
return ("SM_ACTION_API_SM_START");
case HttpTransact::SM_ACTION_REDIRECT_READ:
@@ -438,6 +442,8 @@ HttpDebugNames::get_api_hook_name(TSHttpHookID t)
return "TS_HTTP_SEND_RESPONSE_HDR_HOOK";
case TS_HTTP_REQUEST_TRANSFORM_HOOK:
return "TS_HTTP_REQUEST_TRANSFORM_HOOK";
+ case TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK:
+ return "TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK";
case TS_HTTP_RESPONSE_TRANSFORM_HOOK:
return "TS_HTTP_RESPONSE_TRANSFORM_HOOK";
case TS_HTTP_SELECT_ALT_HOOK:
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index fded2c2..c279f57 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -838,6 +838,48 @@ HttpSM::state_drain_client_request_body(int event, void *data)
}
#endif /* PROXY_DRAIN */
+void
+HttpSM::wait_for_full_body()
+{
+ is_waiting_for_full_body = true;
+ HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler_post);
+ bool chunked = (t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING);
+ int64_t alloc_index;
+ HttpTunnelProducer *p = nullptr;
+
+ // content length is undefined, use default buffer size
+ if (t_state.hdr_info.request_content_length == HTTP_UNDEFINED_CL) {
+ alloc_index = (int)t_state.txn_conf->default_buffer_size_index;
+ if (alloc_index < MIN_CONFIG_BUFFER_SIZE_INDEX || alloc_index > MAX_BUFFER_SIZE_INDEX) {
+ alloc_index = DEFAULT_REQUEST_BUFFER_SIZE_INDEX;
+ }
+ } else {
+ alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length);
+ }
+ MIOBuffer *post_buffer = new_MIOBuffer(alloc_index);
+ IOBufferReader *buf_start = post_buffer->alloc_reader();
+
+ this->_postbuf.init(post_buffer->clone_reader(buf_start));
+
+ // Note: Many browsers, Netscape and IE included send two extra
+ // bytes (CRLF) at the end of the post. We just ignore those
+ // bytes since the sending them is not spec
+
+ // Next order of business if copy the remaining data from the
+ // header buffer into new buffer
+ int64_t post_bytes = chunked ? INT64_MAX : t_state.hdr_info.request_content_length;
+ client_request_body_bytes = post_buffer->write(ua_buffer_reader, chunked ? ua_buffer_reader->read_avail() : post_bytes);
+
+ ua_buffer_reader->consume(client_request_body_bytes);
+ p = tunnel.add_producer(ua_entry->vc, post_bytes, buf_start, &HttpSM::tunnel_handler_post_ua, HT_BUFFER_READ, "ua post buffer");
+ if (chunked) {
+ tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_CHUNKED_CONTENT);
+ }
+ ua_entry->in_tunnel = true;
+ ua_txn->set_inactivity_timeout(HRTIME_SECONDS(t_state.txn_conf->transaction_no_activity_timeout_in));
+ tunnel.tunnel_run(p);
+}
+
int
HttpSM::state_watch_for_client_abort(int event, void *data)
{
@@ -1601,6 +1643,7 @@ HttpSM::handle_api_return()
case HttpTransact::SM_ACTION_API_PRE_REMAP:
case HttpTransact::SM_ACTION_API_POST_REMAP:
case HttpTransact::SM_ACTION_API_READ_REQUEST_HDR:
+ case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE:
case HttpTransact::SM_ACTION_API_OS_DNS:
case HttpTransact::SM_ACTION_API_READ_RESPONSE_HDR:
call_transact_and_set_next_state(nullptr);
@@ -2616,7 +2659,7 @@ HttpSM::main_handler(int event, void *data)
void
HttpSM::tunnel_handler_post_or_put(HttpTunnelProducer *p)
{
- ink_assert(p->vc_type == HT_HTTP_CLIENT);
+ ink_assert(p->vc_type == HT_HTTP_CLIENT || (p->handler_state == HTTP_SM_POST_UA_FAIL && p->vc_type == HT_BUFFER_READ));
HttpTunnelConsumer *c;
// If there is a post transform, remove it's entry from the State
@@ -2715,7 +2758,12 @@ HttpSM::tunnel_handler_post(int event, void *data)
// The tunnel calls this when it is done
int p_handler_state = p->handler_state;
- tunnel_handler_post_or_put(p);
+ if (is_waiting_for_full_body && !this->is_postbuf_valid()) {
+ p_handler_state = HTTP_SM_POST_SERVER_FAIL;
+ }
+ if (p->vc_type != HT_BUFFER_READ) {
+ tunnel_handler_post_or_put(p);
+ }
switch (p_handler_state) {
case HTTP_SM_POST_SERVER_FAIL:
@@ -2725,6 +2773,14 @@ HttpSM::tunnel_handler_post(int event, void *data)
break;
case HTTP_SM_POST_SUCCESS:
// It's time to start reading the response
+ if (is_waiting_for_full_body) {
+ is_waiting_for_full_body = false;
+ is_using_post_buffer = true;
+ client_request_body_bytes = this->postbuf_buffer_avail();
+
+ call_transact_and_set_next_state(HttpTransact::HandleRequestBufferDone);
+ break;
+ }
setup_server_read_response_header();
break;
default:
@@ -3462,7 +3518,7 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p)
// we were setting it again to true but incorrectly in
// the case of a transform
hsm_release_assert(ua_entry->in_tunnel == true);
- if (p->consumer_list.head->vc_type == HT_TRANSFORM) {
+ if (p->consumer_list.head && p->consumer_list.head->vc_type == HT_TRANSFORM) {
hsm_release_assert(post_transform_info.entry->in_tunnel == true);
} else if (server_entry != nullptr) {
hsm_release_assert(server_entry->in_tunnel == true);
@@ -3482,6 +3538,7 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p)
tunnel.local_finish_all(p);
}
}
+
// Initiate another read to watch catch aborts and
// timeouts
ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort;
@@ -3507,6 +3564,7 @@ HttpSM::tunnel_handler_for_partial_post(int event, void * /* data ATS_UNUSED */)
tunnel.reset();
t_state.redirect_info.redirect_in_process = false;
+ is_using_post_buffer = false;
if (post_failed) {
post_failed = false;
@@ -5062,6 +5120,9 @@ HttpSM::do_api_callout_internal()
case HttpTransact::SM_ACTION_API_READ_REQUEST_HDR:
cur_hook_id = TS_HTTP_READ_REQUEST_HDR_HOOK;
break;
+ case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE:
+ cur_hook_id = TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK;
+ break;
case HttpTransact::SM_ACTION_API_OS_DNS:
cur_hook_id = TS_HTTP_OS_DNS_HOOK;
break;
@@ -5302,8 +5363,12 @@ HttpSM::handle_post_failure()
STATE_ENTER(&HttpSM::handle_post_failure, VC_EVENT_NONE);
ink_assert(ua_entry->vc == ua_txn);
- ink_assert(server_entry->eos == true);
+ ink_assert(is_waiting_for_full_body || server_entry->eos == true);
+ if (is_waiting_for_full_body) {
+ call_transact_and_set_next_state(HttpTransact::Forbidden);
+ return;
+ }
// First order of business is to clean up from
// the tunnel
// note: since the tunnel is providing the buffer for a lingering
@@ -5586,7 +5651,8 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type)
// YTS Team, yamsat Plugin
// if redirect_in_process and redirection is enabled add static producer
- if (t_state.redirect_info.redirect_in_process && enable_redirection && (this->_postbuf.postdata_copy_buffer_start != nullptr)) {
+ if (is_using_post_buffer ||
+ (t_state.redirect_info.redirect_in_process && enable_redirection && this->_postbuf.postdata_copy_buffer_start != nullptr)) {
post_redirect = true;
// copy the post data into a new producer buffer for static producer
MIOBuffer *postdata_producer_buffer = new_empty_MIOBuffer();
@@ -7176,6 +7242,7 @@ HttpSM::set_next_state()
case HttpTransact::SM_ACTION_API_PRE_REMAP:
case HttpTransact::SM_ACTION_API_POST_REMAP:
case HttpTransact::SM_ACTION_API_READ_REQUEST_HDR:
+ case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE:
case HttpTransact::SM_ACTION_API_OS_DNS:
case HttpTransact::SM_ACTION_API_SEND_REQUEST_HDR:
case HttpTransact::SM_ACTION_API_READ_CACHE_HDR:
@@ -7586,6 +7653,11 @@ HttpSM::set_next_state()
}
#endif /* PROXY_DRAIN */
+ case HttpTransact::SM_ACTION_WAIT_FOR_FULL_BODY: {
+ wait_for_full_body();
+ break;
+ }
+
case HttpTransact::SM_ACTION_CONTINUE: {
ink_release_assert(!"Not implemented");
break;
@@ -8031,12 +8103,21 @@ HttpSM::find_proto_string(HTTPVersion version) const
void
PostDataBuffers::copy_partial_post_data()
{
- this->postdata_copy_buffer->write(this->ua_buffer_reader);
+ if (post_data_buffer_done) {
+ return;
+ }
Debug("http_redirect", "[PostDataBuffers::copy_partial_post_data] wrote %" PRId64 " bytes to buffers %" PRId64 "",
this->ua_buffer_reader->read_avail(), this->postdata_copy_buffer_start->read_avail());
+ this->postdata_copy_buffer->write(this->ua_buffer_reader);
this->ua_buffer_reader->consume(this->ua_buffer_reader->read_avail());
}
+IOBufferReader *
+PostDataBuffers::get_post_data_buffer_clone_reader()
+{
+ return this->postdata_copy_buffer->clone_reader(this->postdata_copy_buffer_start);
+}
+
// YTS Team, yamsat Plugin
// Allocating the post data buffers
void
@@ -8047,6 +8128,7 @@ PostDataBuffers::init(IOBufferReader *ua_reader)
this->ua_buffer_reader = ua_reader;
if (this->postdata_copy_buffer == nullptr) {
+ this->post_data_buffer_done = false;
ink_assert(this->postdata_copy_buffer_start == nullptr);
this->postdata_copy_buffer = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_4K);
this->postdata_copy_buffer_start = this->postdata_copy_buffer->alloc_reader();
@@ -8067,6 +8149,7 @@ PostDataBuffers::clear()
this->postdata_copy_buffer = nullptr;
this->postdata_copy_buffer_start = nullptr; // deallocated by the buffer
}
+ this->post_data_buffer_done = false;
}
PostDataBuffers::~PostDataBuffers()
diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h
index d98d3a4..4cb535b 100644
--- a/proxy/http/HttpSM.h
+++ b/proxy/http/HttpSM.h
@@ -184,10 +184,27 @@ public:
MIOBuffer *postdata_copy_buffer = nullptr;
IOBufferReader *postdata_copy_buffer_start = nullptr;
IOBufferReader *ua_buffer_reader = nullptr;
+ bool post_data_buffer_done = false;
void clear();
void init(IOBufferReader *ua_reader);
void copy_partial_post_data();
+ IOBufferReader *get_post_data_buffer_clone_reader();
+ void
+ set_post_data_buffer_done(bool done)
+ {
+ post_data_buffer_done = done;
+ }
+ bool
+ get_post_data_buffer_done()
+ {
+ return post_data_buffer_done;
+ }
+ bool
+ is_valid()
+ {
+ return postdata_copy_buffer_start != nullptr;
+ }
~PostDataBuffers();
};
@@ -318,6 +335,10 @@ public:
void disable_redirect();
void postbuf_copy_partial_data();
void postbuf_init(IOBufferReader *ua_reader);
+ void set_postbuf_done(bool done);
+ bool get_postbuf_done();
+ bool is_postbuf_valid();
+ IOBufferReader *get_postbuf_clone_reader();
protected:
int reentrancy_count = 0;
@@ -454,6 +475,8 @@ protected:
void do_drain_request_body();
#endif
+ void wait_for_full_body();
+
virtual void handle_api_return();
void handle_server_setup_error(int event, void *data);
void handle_http_server_open();
@@ -528,6 +551,8 @@ public:
const char *client_cipher_suite = "-";
int server_transact_count = 0;
bool server_connection_is_ssl = false;
+ bool is_waiting_for_full_body = false;
+ bool is_using_post_buffer = false;
TransactionMilestones milestones;
ink_hrtime api_timer = 0;
@@ -731,4 +756,27 @@ HttpSM::postbuf_init(IOBufferReader *ua_reader)
this->_postbuf.init(ua_reader);
}
+inline void
+HttpSM::set_postbuf_done(bool done)
+{
+ this->_postbuf.set_post_data_buffer_done(done);
+}
+
+inline bool
+HttpSM::get_postbuf_done()
+{
+ return this->_postbuf.get_post_data_buffer_done();
+}
+
+inline bool
+HttpSM::is_postbuf_valid()
+{
+ return this->_postbuf.is_valid();
+}
+
+inline IOBufferReader *
+HttpSM::get_postbuf_clone_reader()
+{
+ return this->_postbuf.get_post_data_buffer_clone_reader();
+}
#endif
diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc
index 5f16743..9af7900 100644
--- a/proxy/http/HttpTransact.cc
+++ b/proxy/http/HttpTransact.cc
@@ -1123,77 +1123,81 @@ HttpTransact::HandleRequest(State *s)
{
TxnDebug("http_trans", "START HttpTransact::HandleRequest");
- ink_assert(!s->hdr_info.server_request.valid());
+ if (!s->request_data.hdr) {
+ ink_assert(!s->hdr_info.server_request.valid());
- HTTP_INCREMENT_DYN_STAT(http_incoming_requests_stat);
+ HTTP_INCREMENT_DYN_STAT(http_incoming_requests_stat);
- if (s->client_info.port_attribute == HttpProxyPort::TRANSPORT_SSL) {
- HTTP_INCREMENT_DYN_STAT(https_incoming_requests_stat);
- }
-
- ///////////////////////////////////////////////
- // if request is bad, return error response //
- ///////////////////////////////////////////////
-
- if (!(is_request_valid(s, &s->hdr_info.client_request))) {
- HTTP_INCREMENT_DYN_STAT(http_invalid_client_requests_stat);
- TxnDebug("http_seq", "[HttpTransact::HandleRequest] request invalid.");
- s->next_action = SM_ACTION_SEND_ERROR_CACHE_NOOP;
- // s->next_action = HttpTransact::PROXY_INTERNAL_CACHE_NOOP;
- return;
- }
- TxnDebug("http_seq", "[HttpTransact::HandleRequest] request valid.");
+ if (s->client_info.port_attribute == HttpProxyPort::TRANSPORT_SSL) {
+ HTTP_INCREMENT_DYN_STAT(https_incoming_requests_stat);
+ }
- if (is_debug_tag_set("http_chdr_describe")) {
- obj_describe(s->hdr_info.client_request.m_http, true);
- }
+ ///////////////////////////////////////////////
+ // if request is bad, return error response //
+ ///////////////////////////////////////////////
- // at this point we are guaranteed that the request is good and acceptable.
- // initialize some state variables from the request (client version,
- // client keep-alive, cache action, etc.
- initialize_state_variables_from_request(s, &s->hdr_info.client_request);
+ if (!(is_request_valid(s, &s->hdr_info.client_request))) {
+ HTTP_INCREMENT_DYN_STAT(http_invalid_client_requests_stat);
+ TxnDebug("http_seq", "[HttpTransact::HandleRequest] request invalid.");
+ s->next_action = SM_ACTION_SEND_ERROR_CACHE_NOOP;
+ // s->next_action = HttpTransact::PROXY_INTERNAL_CACHE_NOOP;
+ return;
+ }
+ TxnDebug("http_seq", "[HttpTransact::HandleRequest] request valid.");
+
+ if (is_debug_tag_set("http_chdr_describe")) {
+ obj_describe(s->hdr_info.client_request.m_http, true);
+ }
+ // at this point we are guaranteed that the request is good and acceptable.
+ // initialize some state variables from the request (client version,
+ // client keep-alive, cache action, etc.
+ initialize_state_variables_from_request(s, &s->hdr_info.client_request);
+ // The following chunk of code will limit the maximum number of websocket connections (TS-3659)
+ if (s->is_upgrade_request && s->is_websocket && s->http_config_param->max_websocket_connections >= 0) {
+ int64_t val = 0;
+ HTTP_READ_DYN_SUM(http_websocket_current_active_client_connections_stat, val);
+ if (val >= s->http_config_param->max_websocket_connections) {
+ s->is_websocket = false; // unset to avoid screwing up stats.
+ TxnDebug("http_trans", "Rejecting websocket connection because the limit has been exceeded");
+ bootstrap_state_variables_from_request(s, &s->hdr_info.client_request);
+ build_error_response(s, HTTP_STATUS_SERVICE_UNAVAILABLE, "WebSocket Connection Limit Exceeded", nullptr);
+ TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr);
+ }
+ }
- // The following chunk of code will limit the maximum number of websocket connections (TS-3659)
- if (s->is_upgrade_request && s->is_websocket && s->http_config_param->max_websocket_connections >= 0) {
- int64_t val = 0;
- HTTP_READ_DYN_SUM(http_websocket_current_active_client_connections_stat, val);
- if (val >= s->http_config_param->max_websocket_connections) {
- s->is_websocket = false; // unset to avoid screwing up stats.
- TxnDebug("http_trans", "Rejecting websocket connection because the limit has been exceeded");
+ // The following code is configurable to allow a user to control the max post size (TS-3631)
+ if (s->http_config_param->max_post_size > 0 && s->hdr_info.request_content_length > 0 &&
+ s->hdr_info.request_content_length > s->http_config_param->max_post_size) {
+ TxnDebug("http_trans", "Max post size %" PRId64 " Client tried to post a body that was too large.",
+ s->http_config_param->max_post_size);
+ HTTP_INCREMENT_DYN_STAT(http_post_body_too_large);
bootstrap_state_variables_from_request(s, &s->hdr_info.client_request);
- build_error_response(s, HTTP_STATUS_SERVICE_UNAVAILABLE, "WebSocket Connection Limit Exceeded", nullptr);
+ build_error_response(s, HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE, "Request Entity Too Large", "request#entity_too_large");
+ s->squid_codes.log_code = SQUID_LOG_ERR_POST_ENTITY_TOO_LARGE;
TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr);
}
- }
-
- // The following code is configurable to allow a user to control the max post size (TS-3631)
- if (s->http_config_param->max_post_size > 0 && s->hdr_info.request_content_length > 0 &&
- s->hdr_info.request_content_length > s->http_config_param->max_post_size) {
- TxnDebug("http_trans", "Max post size %" PRId64 " Client tried to post a body that was too large.",
- s->http_config_param->max_post_size);
- HTTP_INCREMENT_DYN_STAT(http_post_body_too_large);
- bootstrap_state_variables_from_request(s, &s->hdr_info.client_request);
- build_error_response(s, HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE, "Request Entity Too Large", "request#entity_too_large");
- s->squid_codes.log_code = SQUID_LOG_ERR_POST_ENTITY_TOO_LARGE;
- TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr);
- }
- // The following chunk of code allows you to disallow post w/ expect 100-continue (TS-3459)
- if (s->hdr_info.request_content_length && s->http_config_param->disallow_post_100_continue) {
- MIMEField *expect = s->hdr_info.client_request.field_find(MIME_FIELD_EXPECT, MIME_LEN_EXPECT);
-
- if (expect != nullptr) {
- const char *expect_hdr_val = nullptr;
- int expect_hdr_val_len = 0;
- expect_hdr_val = expect->value_get(&expect_hdr_val_len);
- if (ptr_len_casecmp(expect_hdr_val, expect_hdr_val_len, HTTP_VALUE_100_CONTINUE, HTTP_LEN_100_CONTINUE) == 0) {
- // Let's error out this request.
- TxnDebug("http_trans", "Client sent a post expect: 100-continue, sending 405.");
- HTTP_INCREMENT_DYN_STAT(disallowed_post_100_continue);
- build_error_response(s, HTTP_STATUS_METHOD_NOT_ALLOWED, "Method Not Allowed", "request#method_unsupported");
- TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr);
+ // The following chunk of code allows you to disallow post w/ expect 100-continue (TS-3459)
+ if (s->hdr_info.request_content_length && s->http_config_param->disallow_post_100_continue) {
+ MIMEField *expect = s->hdr_info.client_request.field_find(MIME_FIELD_EXPECT, MIME_LEN_EXPECT);
+
+ if (expect != nullptr) {
+ const char *expect_hdr_val = nullptr;
+ int expect_hdr_val_len = 0;
+ expect_hdr_val = expect->value_get(&expect_hdr_val_len);
+ if (ptr_len_casecmp(expect_hdr_val, expect_hdr_val_len, HTTP_VALUE_100_CONTINUE, HTTP_LEN_100_CONTINUE) == 0) {
+ // Let's error out this request.
+ TxnDebug("http_trans", "Client sent a post expect: 100-continue, sending 405.");
+ HTTP_INCREMENT_DYN_STAT(disallowed_post_100_continue);
+ build_error_response(s, HTTP_STATUS_METHOD_NOT_ALLOWED, "Method Not Allowed", "request#method_unsupported");
+ TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr);
+ }
}
}
+ if (s->txn_conf->request_buffer_enabled &&
+ (s->hdr_info.request_content_length > 0 || s->client_info.transfer_encoding == CHUNKED_ENCODING)) {
+ TRANSACT_RETURN(SM_ACTION_WAIT_FOR_FULL_BODY, nullptr);
+ }
}
// Cache lookup or not will be decided later at DecideCacheLookup().
@@ -1307,6 +1311,12 @@ HttpTransact::HandleRequest(State *s)
}
void
+HttpTransact::HandleRequestBufferDone(State *s)
+{
+ TRANSACT_RETURN(SM_ACTION_REQUEST_BUFFER_READ_COMPLETE, HttpTransact::HandleRequest);
+}
+
+void
HttpTransact::setup_plugin_request_intercept(State *s)
{
ink_assert(s->state_machine->plugin_tunnel != nullptr);
diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h
index 3b09a71..3d30058 100644
--- a/proxy/http/HttpTransact.h
+++ b/proxy/http/HttpTransact.h
@@ -425,6 +425,8 @@ public:
SM_ACTION_DRAIN_REQUEST_BODY,
#endif /* PROXY_DRAIN */
+ SM_ACTION_WAIT_FOR_FULL_BODY,
+ SM_ACTION_REQUEST_BUFFER_READ_COMPLETE,
SM_ACTION_SERVE_FROM_CACHE,
SM_ACTION_SERVER_READ,
SM_ACTION_SERVER_PARSE_NEXT_HDR,
@@ -957,6 +959,7 @@ public:
static void PerformRemap(State *s);
static void ModifyRequest(State *s);
static void HandleRequest(State *s);
+ static void HandleRequestBufferDone(State *s);
static bool handleIfRedirect(State *s);
static void StartAccessControl(State *s);
diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc
index c5f1fa5..606e1b0 100644
--- a/proxy/http/HttpTunnel.cc
+++ b/proxy/http/HttpTunnel.cc
@@ -757,7 +757,6 @@ void
HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg)
{
Debug("http_tunnel", "tunnel_run started, p_arg is %s", p_arg ? "provided" : "NULL");
-
if (p_arg) {
producer_run(p_arg);
} else {
@@ -876,7 +875,6 @@ HttpTunnel::producer_run(HttpTunnelProducer *p)
// Do the IO on the consumers first so
// data doesn't disappear out from
// under the tunnel
- ink_release_assert(p->num_consumers > 0);
for (c = p->consumer_list.head; c;) {
// Create a reader for each consumer. The reader allows
// us to implement skip bytes
@@ -952,15 +950,20 @@ HttpTunnel::producer_run(HttpTunnelProducer *p)
// YTS Team, yamsat Plugin
// Allocate and copy partial POST data to buffers. Check for the various parameters
// including the maximum configured post data size
- if (p->alive && sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection && (p->vc_type == HT_HTTP_CLIENT)) {
+ if ((p->vc_type == HT_BUFFER_READ && sm->is_postbuf_valid()) ||
+ (p->alive && sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection && p->vc_type == HT_HTTP_CLIENT)) {
Debug("http_redirect", "[HttpTunnel::producer_run] client post: %" PRId64 " max size: %" PRId64 "",
p->buffer_start->read_avail(), HttpConfig::m_master.post_copy_size);
// (note that since we are not dechunking POST, this is the chunked size if chunked)
if (p->buffer_start->read_avail() > HttpConfig::m_master.post_copy_size) {
- Debug("http_redirect", "[HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64 " limit=%" PRId64 "",
- p->buffer_start->read_avail(), HttpConfig::m_master.post_copy_size);
+ Warning("http_redirect, [HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64 " limit=%" PRId64 "",
+ p->buffer_start->read_avail(), HttpConfig::m_master.post_copy_size);
sm->disable_redirect();
+ if (p->vc_type == HT_BUFFER_READ) {
+ producer_handler(VC_EVENT_ERROR, p);
+ return;
+ }
} else {
sm->postbuf_copy_partial_data();
}
@@ -993,8 +996,7 @@ HttpTunnel::producer_run(HttpTunnelProducer *p)
// p->chunked_handler.skip_bytes);
producer_handler(VC_EVENT_READ_READY, p);
- if (!p->chunked_handler.chunked_reader->read_avail() && sm->redirection_tries > 0 &&
- p->vc_type == HT_HTTP_CLIENT) { // read_avail() == 0
+ if (sm->get_postbuf_done() && p->vc_type == HT_HTTP_CLIENT) { // read_avail() == 0
// [bug 2579251]
// Ugh, this is horrible but in the redirect case they are running a the tunnel again with the
// now closed/empty producer to trigger PRECOMPLETE. If the POST was chunked, producer_n is set
@@ -1162,17 +1164,24 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p)
// YTS Team, yamsat Plugin
// Copy partial POST data to buffers. Check for the various parameters including
// the maximum configured post data size
- if (sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection &&
- (event == VC_EVENT_READ_READY || event == VC_EVENT_READ_COMPLETE) && (p->vc_type == HT_HTTP_CLIENT)) {
+ if ((p->vc_type == HT_BUFFER_READ && sm->is_postbuf_valid()) ||
+ (sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection &&
+ (event == VC_EVENT_READ_READY || event == VC_EVENT_READ_COMPLETE) && p->vc_type == HT_HTTP_CLIENT)) {
Debug("http_redirect", "[HttpTunnel::producer_handler] [%s %s]", p->name, HttpDebugNames::get_event_name(event));
if ((sm->postbuf_buffer_avail() + sm->postbuf_reader_avail()) > HttpConfig::m_master.post_copy_size) {
- Debug("http_redirect", "[HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64
- " reader_avail=%" PRId64 " limit=%" PRId64 "",
- sm->postbuf_buffer_avail(), sm->postbuf_reader_avail(), HttpConfig::m_master.post_copy_size);
+ Warning("http_redirect, [HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64
+ " reader_avail=%" PRId64 " limit=%" PRId64 "",
+ sm->postbuf_buffer_avail(), sm->postbuf_reader_avail(), HttpConfig::m_master.post_copy_size);
sm->disable_redirect();
+ if (p->vc_type == HT_BUFFER_READ) {
+ event = VC_EVENT_ERROR;
+ }
} else {
sm->postbuf_copy_partial_data();
+ if (event == VC_EVENT_READ_COMPLETE || event == HTTP_TUNNEL_EVENT_PRECOMPLETE || event == VC_EVENT_EOS) {
+ sm->set_postbuf_done(true);
+ }
}
} // end of added logic for partial copy of POST
diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h
index d98c32d..ca15219 100644
--- a/proxy/http/HttpTunnel.h
+++ b/proxy/http/HttpTunnel.h
@@ -66,14 +66,7 @@ struct HttpTunnelProducer;
typedef int (HttpSM::*HttpProducerHandler)(int event, HttpTunnelProducer *p);
typedef int (HttpSM::*HttpConsumerHandler)(int event, HttpTunnelConsumer *c);
-enum HttpTunnelType_t {
- HT_HTTP_SERVER,
- HT_HTTP_CLIENT,
- HT_CACHE_READ,
- HT_CACHE_WRITE,
- HT_TRANSFORM,
- HT_STATIC,
-};
+enum HttpTunnelType_t { HT_HTTP_SERVER, HT_HTTP_CLIENT, HT_CACHE_READ, HT_CACHE_WRITE, HT_TRANSFORM, HT_STATIC, HT_BUFFER_READ };
enum TunnelChunkingAction_t {
TCA_CHUNK_CONTENT,
--
To stop receiving notification emails like this one, please contact
bcall@apache.org.
[trafficserver] 02/02: Adding slow_post_test
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bcall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 88c74e4a09efc0d39f1d6d5292859bf9949bf973
Author: Zizhong Zhang <zi...@linkedin.com>
AuthorDate: Tue Jan 2 12:29:56 2018 -0800
Adding slow_post_test
---
tests/gold_tests/slow_post/gold/200.gold | 1 +
tests/gold_tests/slow_post/slow_post.test.py | 73 +++++++++++++
tests/gold_tests/slow_post/slow_post_client.py | 59 ++++++++++
tests/tools/plugins/request_buffer.c | 144 +++++++++++++++++++++++++
4 files changed, 277 insertions(+)
diff --git a/tests/gold_tests/slow_post/gold/200.gold b/tests/gold_tests/slow_post/gold/200.gold
new file mode 100644
index 0000000..08839f6
--- /dev/null
+++ b/tests/gold_tests/slow_post/gold/200.gold
@@ -0,0 +1 @@
+200
diff --git a/tests/gold_tests/slow_post/slow_post.test.py b/tests/gold_tests/slow_post/slow_post.test.py
new file mode 100644
index 0000000..906a87f
--- /dev/null
+++ b/tests/gold_tests/slow_post/slow_post.test.py
@@ -0,0 +1,73 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+
+class SlowPostAttack:
+ def __init__(cls):
+ Test.Summary = 'Test how ATS handles the slow-post attack'
+ cls._origin_max_connections = 3
+ cls._slow_post_client = 'slow_post_client.py'
+ cls.setupOriginServer()
+ cls.setupTS()
+ cls._ts.Setup.CopyAs(cls._slow_post_client, Test.RunDirectory)
+
+ def setupOriginServer(self):
+ self._server = Test.MakeOriginServer("server")
+ request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+ response_header = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n",
+ "timestamp": "1469733493.993", "body": ""}
+ self._server.addResponse("sessionlog.json", request_header, response_header)
+ request_header2 = {"headers": "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nHost: www.example.com\r\nConnection: keep-alive\r\n\r\n",
+ "timestamp": "1469733493.993", "body": "a\r\na\r\na\r\n\r\n"}
+ response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n",
+ "timestamp": "1469733493.993", "body": ""}
+ self._server.addResponse("sessionlog.json", request_header2, response_header2)
+
+ def setupTS(self):
+ self._ts = Test.MakeATSProcess("ts", select_ports=False)
+ self._ts.Disk.remap_config.AddLine(
+ 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port)
+ )
+ # This plugin can enable request buffer for POST.
+ self._ts.Disk.plugin_config.AddLine(
+ 'request_buffer.so'
+ )
+ Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'request_buffer.c'), self._ts)
+ self._ts.Disk.records_config.update({
+ 'proxy.config.diags.debug.enabled': 1,
+ 'proxy.config.diags.debug.tags': 'http',
+ 'proxy.config.http.origin_max_connections': self._origin_max_connections,
+ # Disable queueing when connection reaches limit
+ 'proxy.config.http.origin_max_connections_queue': 0,
+ })
+
+ def run(self):
+ tr = Test.AddTestRun()
+ tr.Processes.Default.Command = 'python3 {0} -p {1} -c {2}'.format(
+ self._slow_post_client, self._ts.Variables.port, self._origin_max_connections)
+ tr.Processes.Default.ReturnCode = 0
+ tr.Processes.Default.StartBefore(self._server)
+ tr.Processes.Default.StartBefore(Test.Processes.ts)
+ tr.Processes.Default.Streams.stdout = "gold/200.gold"
+
+
+Test.Summary = 'Test how ATS handles the slow-post attack'
+slowPostAttack = SlowPostAttack()
+slowPostAttack.run()
diff --git a/tests/gold_tests/slow_post/slow_post_client.py b/tests/gold_tests/slow_post/slow_post_client.py
new file mode 100644
index 0000000..a132a67
--- /dev/null
+++ b/tests/gold_tests/slow_post/slow_post_client.py
@@ -0,0 +1,59 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import threading
+import requests
+import argparse
+
+
+def gen(slow_time):
+ for _ in range(slow_time):
+ yield b'a'
+ time.sleep(1)
+
+
+def slow_post(port, slow_time):
+ requests.post('http://127.0.0.1:{0}/'.format(port, ), data=gen(slow_time))
+
+
+def makerequest(port, connection_limit):
+ client_timeout = 3
+ for i in range(connection_limit):
+ t = threading.Thread(target=slow_post, args=(port, client_timeout + 10))
+ t.daemon = True
+ t.start()
+ time.sleep(1)
+ r = requests.get('http://127.0.0.1:{0}/'.format(port,))
+ print(r.status_code)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--port", "-p",
+ type=int,
+ help="Port to use")
+ parser.add_argument("--connectionlimit", "-c",
+ type=int,
+ help="connection limit")
+ args = parser.parse_args()
+ makerequest(args.port, args.connectionlimit)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/tools/plugins/request_buffer.c b/tests/tools/plugins/request_buffer.c
new file mode 100644
index 0000000..7e6df14
--- /dev/null
+++ b/tests/tools/plugins/request_buffer.c
@@ -0,0 +1,144 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ts/ts.h"
+
+#define PLUGIN_NAME "request_buffer"
+
+#define TS_NULL_MUTEX NULL
+
+static char *
+request_body_get(TSHttpTxn txnp, int *len)
+{
+ char *ret = NULL;
+ TSIOBufferReader post_buffer_reader = TSHttpTxnPostBufferReaderGet(txnp);
+ int64_t read_avail = TSIOBufferReaderAvail(post_buffer_reader);
+ if (read_avail == 0) {
+ TSIOBufferReaderFree(post_buffer_reader);
+ return NULL;
+ }
+
+ ret = (char *)TSmalloc(sizeof(char) * read_avail);
+
+ int64_t consumed = 0;
+ int64_t data_len = 0;
+ const char *char_data = NULL;
+ TSIOBufferBlock block = TSIOBufferReaderStart(post_buffer_reader);
+ while (block != NULL) {
+ char_data = TSIOBufferBlockReadStart(block, post_buffer_reader, &data_len);
+ memcpy(ret + consumed, char_data, data_len);
+ consumed += data_len;
+ block = TSIOBufferBlockNext(block);
+ }
+ TSIOBufferReaderFree(post_buffer_reader);
+
+ *len = (int)consumed;
+ return ret;
+}
+
+static int
+request_buffer_plugin(TSCont contp, TSEvent event, void *edata)
+{
+ TSDebug(PLUGIN_NAME, "request_buffer_plugin starting, event[%d]", event);
+ TSHttpTxn txnp = (TSHttpTxn)(edata);
+ if (event == TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE) {
+ int len = 0;
+ char *body = request_body_get(txnp, &len);
+ TSDebug(PLUGIN_NAME, "request_buffer_plugin gets the request body with length[%d]", len);
+ TSfree(body);
+ TSContDestroy(contp);
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+}
+
+bool
+is_post_request(TSHttpTxn txnp)
+{
+ TSMLoc req_loc;
+ TSMBuffer req_bufp;
+ if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_loc) == TS_ERROR) {
+ TSError("Error while retrieving client request header\n");
+ return false;
+ }
+ int method_len = 0;
+ const char *method = TSHttpHdrMethodGet(req_bufp, req_loc, &method_len);
+ if (method_len != (int)strlen(TS_HTTP_METHOD_POST) || strncasecmp(method, TS_HTTP_METHOD_POST, method_len) != 0) {
+ TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc);
+ return false;
+ }
+ TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc);
+ return true;
+}
+
+static int
+global_plugin(TSCont contp, TSEvent event, void *edata)
+{
+ TSDebug(PLUGIN_NAME, "transform_plugin starting");
+ TSHttpTxn txnp = (TSHttpTxn)edata;
+
+ switch (event) {
+ case TS_EVENT_HTTP_READ_REQUEST_HDR:
+ if (is_post_request(txnp)) {
+ TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, 1);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TSContCreate(request_buffer_plugin, TSMutexCreate()));
+ }
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+void
+TSPluginInit(int argc, const char *argv[])
+{
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = PLUGIN_NAME;
+ info.vendor_name = "Apache Software Foundation";
+ info.support_email = "dev@trafficserver.apache.org";
+
+ if (TSPluginRegister(&info) != TS_SUCCESS) {
+ TSDebug(PLUGIN_NAME, "[%s] Plugin registration failed", PLUGIN_NAME);
+
+ goto Lerror;
+ }
+
+ /* This is call we could use if we need to protect global data */
+ /* TSReleaseAssert ((mutex = TSMutexCreate()) != TS_NULL_MUTEX); */
+
+ TSMutex mutex = TS_NULL_MUTEX;
+ TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(global_plugin, mutex));
+ TSDebug(PLUGIN_NAME, "[%s] Plugin registration succeeded", PLUGIN_NAME);
+ return;
+
+Lerror:
+ TSDebug(PLUGIN_NAME, "[%s] Plugin disabled", PLUGIN_NAME);
+}
--
To stop receiving notification emails like this one, please contact
bcall@apache.org.