You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2020/05/13 20:51:56 UTC
[trafficserver] branch 9.0.x updated: Enforce Active Connection
limits (#6754)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/9.0.x by this push:
new 5790dfa Enforce Active Connection limits (#6754)
5790dfa is described below
commit 5790dfaf0e764775dea73514fc7b42d4478684e9
Author: Sudheer Vinukonda <su...@apache.org>
AuthorDate: Tue May 12 11:13:00 2020 -0700
Enforce Active Connection limits (#6754)
* Enforce Active Connection limits
1. Throttle connections when there's no room in active conn queue
2. Adjust manage_active_queue() to not fail when the conn is already in active queue
3. Return true for PluginVC (dummy connection) add_to_active_queue
4. Metrics for throttling
5. Allow to disable active connection tracking
6. Doc updates
(cherry picked from commit deee3efbbe64e19d31a953e222edbffb8ab06954)
---
doc/admin-guide/files/records.config.en.rst | 16 ++++++++++++++++
.../monitoring/statistics/core/network-io.en.rst | 11 ++++++++++-
iocore/net/Net.cc | 2 ++
iocore/net/P_Net.h | 1 +
iocore/net/UnixNet.cc | 15 +++++++++++++--
proxy/PluginVC.cc | 2 +-
proxy/http/Http1ClientSession.cc | 7 ++++++-
proxy/http2/Http2ConnectionState.cc | 8 +++++++-
8 files changed, 56 insertions(+), 6 deletions(-)
diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
index 0ff8556..0cb2509 100644
--- a/doc/admin-guide/files/records.config.en.rst
+++ b/doc/admin-guide/files/records.config.en.rst
@@ -409,6 +409,22 @@ Network
handled. This should be tuned according to your memory size, and expected
work load. If this is set to 0, the throttling logic is disabled.
+.. ts:cv:: CONFIG proxy.config.net.max_connections_in INT 30000
+
+ The total number of client connections that the :program:`traffic_server`
+ can handle simultaneously. This should be tuned according to your memory size,
+ and expected work load (network, cpu etc). This limit includes both keepalive
+ and active client connections that :program:`traffic_server` can handle at
+ any given instant.
+
+.. ts:cv:: CONFIG proxy.config.net.max_active_connections_in INT 10000
+
+ The total number of active client connections that the |TS| can handle
+ simultaneously. This should be tuned according to your memory size,
+ and expected work load (network, cpu etc). If this is set to 0, active
+ connection tracking is disabled and active connections have no separate
+ limit and the total connections follow `proxy.config.net.connections_throttle`
+
.. ts:cv:: CONFIG proxy.config.net.default_inactivity_timeout INT 86400
:reloadable:
diff --git a/doc/admin-guide/monitoring/statistics/core/network-io.en.rst b/doc/admin-guide/monitoring/statistics/core/network-io.en.rst
index 5393086..8b83253 100644
--- a/doc/admin-guide/monitoring/statistics/core/network-io.en.rst
+++ b/doc/admin-guide/monitoring/statistics/core/network-io.en.rst
@@ -60,8 +60,17 @@ Network I/O
.. ts:stat:: global proxy.process.net.connections_currently_open integer
:type: counter
+.. ts:stat:: global proxy.process.net.connections_throttled_in integer
+ :type: counter
+
+.. ts:stat:: global proxy.process.net.connections_throttled_out integer
+ :type: counter
+
+.. ts:stat:: global proxy.process.net.max.active.connections_throttled_in integer
+ :type: counter
+
.. ts:stat:: global proxy.process.net.default_inactivity_timeout_applied integer
-.. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_count integer
+.. ts:stat:: global proxy.process.net.default_inactivity_timeout_count integer
.. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_total integer
.. ts:stat:: global proxy.process.net.inactivity_cop_lock_acquire_failure integer
.. ts:stat:: global proxy.process.net.net_handler_run integer
diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc
index f5bbb88..1813c0d 100644
--- a/iocore/net/Net.cc
+++ b/iocore/net/Net.cc
@@ -140,6 +140,8 @@ register_net_stats()
(int)net_connections_throttled_in_stat, RecRawStatSyncSum);
RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT,
(int)net_connections_throttled_out_stat, RecRawStatSyncSum);
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.max.active.connections_throttled_in", RECD_INT, RECP_PERSISTENT,
+ (int)net_connections_max_active_throttled_in_stat, RecRawStatSyncSum);
}
void
diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h
index 15a55bb..9a749f9 100644
--- a/iocore/net/P_Net.h
+++ b/iocore/net/P_Net.h
@@ -57,6 +57,7 @@ enum Net_Stats {
net_tcp_accept_stat,
net_connections_throttled_in_stat,
net_connections_throttled_out_stat,
+ net_connections_max_active_throttled_in_stat,
Net_Stat_Count
};
diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc
index 7690077..1d91079 100644
--- a/iocore/net/UnixNet.cc
+++ b/iocore/net/UnixNet.cc
@@ -562,6 +562,11 @@ NetHandler::manage_active_queue(bool ignore_queue_size = false)
max_connections_per_thread_in, max_connections_active_per_thread_in, total_connections_in, active_queue_size,
keep_alive_queue_size);
+ if (!max_connections_active_per_thread_in) {
+ // active queue has no max
+ return true;
+ }
+
if (ignore_queue_size == false && max_connections_active_per_thread_in > active_queue_size) {
return true;
}
@@ -722,16 +727,22 @@ NetHandler::add_to_active_queue(UnixNetVConnection *vc)
max_connections_per_thread_in, active_queue_size, keep_alive_queue_size);
ink_assert(mutex->thread_holding == this_ethread());
+ bool active_queue_full = false;
+
// if active queue is over size then close inactive connections
if (manage_active_queue() == false) {
- // there is no room left in the queue
- return false;
+ active_queue_full = true;
}
if (active_queue.in(vc)) {
// already in the active queue, move the head
active_queue.remove(vc);
} else {
+ if (active_queue_full) {
+ // there is no room left in the queue
+ NET_SUM_DYN_STAT(net_connections_max_active_throttled_in_stat, 1);
+ return false;
+ }
// in the keep-alive queue or no queue, new to this queue
remove_from_keep_alive_queue(vc);
++active_queue_size;
diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc
index f6cc427..126b4f0 100644
--- a/proxy/PluginVC.cc
+++ b/proxy/PluginVC.cc
@@ -926,7 +926,7 @@ bool
PluginVC::add_to_active_queue()
{
// do nothing
- return false;
+ return true;
}
SOCKET
diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc
index 8f825dc..0c402bf 100644
--- a/proxy/http/Http1ClientSession.cc
+++ b/proxy/http/Http1ClientSession.cc
@@ -465,6 +465,12 @@ Http1ClientSession::new_transaction()
return;
}
+ if (!client_vc->add_to_active_queue()) {
+ // no room in the active queue close the connection
+ this->do_io_close();
+ return;
+ }
+
// Defensive programming, make sure nothing persists across
// connection re-use
half_close = false;
@@ -474,7 +480,6 @@ Http1ClientSession::new_transaction()
trans.set_proxy_ssn(this);
transact_count++;
- client_vc->add_to_active_queue();
trans.new_transaction(read_from_early_data > 0 ? true : false);
}
diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc
index e8d59a7..3c1654e 100644
--- a/proxy/http2/Http2ConnectionState.cc
+++ b/proxy/http2/Http2ConnectionState.cc
@@ -1150,6 +1150,13 @@ Http2ConnectionState::state_closed(int event, void *edata)
Http2Stream *
Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error)
{
+ // first check if we've hit the active connection limit
+ if (!ua_session->get_netvc()->add_to_active_queue()) {
+ error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_NO_ERROR,
+ "refused to create new stream, maxed out active connections");
+ return nullptr;
+ }
+
// In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client
if (ua_session->get_half_close_local_flag()) {
error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM,
@@ -1223,7 +1230,6 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error)
new_stream->mutex = new_ProxyMutex();
new_stream->is_first_transaction_flag = get_stream_requests() == 0;
increment_stream_requests();
- ua_session->get_netvc()->add_to_active_queue();
return new_stream;
}