You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by sh...@apache.org on 2018/02/07 20:49:14 UTC
[trafficserver] branch master updated: Make throttling feature more
useful.
This is an automated email from the ASF dual-hosted git repository.
shinrich pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 4bfaf36 Make throttling feature more useful.
4bfaf36 is described below
commit 4bfaf364529d49f576e2f3d27b3388364ca4606e
Author: Susan Hinrichs <sh...@apache.org>
AuthorDate: Wed Feb 7 01:04:22 2018 +0000
Make throttling feature more useful.
---
iocore/net/Net.cc | 5 ++++
iocore/net/P_Net.h | 2 ++
iocore/net/P_UnixNet.h | 46 ++++++------------------------------
iocore/net/UnixNet.cc | 1 -
iocore/net/UnixNetAccept.cc | 51 ++++++++++++++++++++++++++++------------
iocore/net/UnixNetVConnection.cc | 12 +++-------
proxy/http/HttpSM.cc | 3 +++
proxy/http/HttpSM.h | 1 -
8 files changed, 56 insertions(+), 65 deletions(-)
diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc
index 94ed2aa..ff6b0cb 100644
--- a/iocore/net/Net.cc
+++ b/iocore/net/Net.cc
@@ -112,6 +112,11 @@ register_net_stats()
RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT,
static_cast<int>(net_tcp_accept_stat), RecRawStatSyncSum);
NET_CLEAR_DYN_STAT(net_tcp_accept_stat);
+
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_in", RECD_INT, RECP_PERSISTENT,
+ (int)net_connections_throttled_in_stat, RecRawStatSyncSum);
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT,
+ (int)net_connections_throttled_out_stat, RecRawStatSyncSum);
}
void
diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h
index 9fecf19..fea3250 100644
--- a/iocore/net/P_Net.h
+++ b/iocore/net/P_Net.h
@@ -56,6 +56,8 @@ enum Net_Stats {
net_fastopen_attempts_stat,
net_fastopen_successes_stat,
net_tcp_accept_stat,
+ net_connections_throttled_in_stat,
+ net_connections_throttled_out_stat,
Net_Stat_Count
};
diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h
index 6c69bd3..f4d5d8e 100644
--- a/iocore/net/P_UnixNet.h
+++ b/iocore/net/P_UnixNet.h
@@ -142,6 +142,7 @@ extern int http_accept_port_number;
// also the 'throttle connect headroom'
#define EMERGENCY_THROTTLE 16
+#define THROTTLE_AT_ONCE 5
#define HYPER_EMERGENCY_THROTTLE 6
#define NET_THROTTLE_ACCEPT_HEADROOM 1.1 // 10%
@@ -404,60 +405,27 @@ check_shedding_warning()
}
TS_INLINE bool
-emergency_throttle(ink_hrtime now)
-{
- return (bool)(emergency_throttle_time > now);
-}
-
-TS_INLINE bool
-check_net_throttle(ThrottleType t, ink_hrtime now)
+check_net_throttle(ThrottleType t)
{
int connections = net_connections_to_throttle(t);
if (connections >= net_connections_throttle)
return true;
- if (emergency_throttle(now))
- return true;
-
return false;
}
TS_INLINE void
-check_throttle_warning()
+check_throttle_warning(ThrottleType type)
{
ink_hrtime t = Thread::get_hrtime();
if (t - last_throttle_warning > NET_THROTTLE_MESSAGE_EVERY) {
last_throttle_warning = t;
- RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR, "too many connections, throttling");
- }
-}
-
-//
-// Emergency throttle when we are close to exhausting file descriptors.
-// Block all accepts or connects for N seconds where N
-// is the amount into the emergency fd stack squared
-// (e.g. on the last file descriptor we have 14 * 14 = 196 seconds
-// of emergency throttle).
-//
-// Hyper Emergency throttle when we are very close to exhausting file
-// descriptors.
-//
-TS_INLINE bool
-check_emergency_throttle(Connection &con)
-{
- int fd = con.fd;
- int emergency = fds_limit - EMERGENCY_THROTTLE;
- if (fd > emergency) {
- int over = fd - emergency;
- emergency_throttle_time = Thread::get_hrtime() + (over * over) * HRTIME_SECOND;
- RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR, "too many open file descriptors, emergency throttling");
- int hyper_emergency = fds_limit - HYPER_EMERGENCY_THROTTLE;
- if (fd > hyper_emergency) {
- return true;
- }
+ int connections = net_connections_to_throttle(type);
+ RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR,
+ "too many connections, throttling. connection_type=%s, current_connections=%d, net_connections_throttle=%d",
+ type == ACCEPT ? "ACCEPT" : "CONNECT", connections, net_connections_throttle);
}
- return false;
}
TS_INLINE int
diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc
index 2fe0e3f..880f490 100644
--- a/iocore/net/UnixNet.cc
+++ b/iocore/net/UnixNet.cc
@@ -25,7 +25,6 @@
ink_hrtime last_throttle_warning;
ink_hrtime last_shedding_warning;
-ink_hrtime emergency_throttle_time;
int net_connections_throttle;
bool net_memory_throttle = false;
int fds_throttle;
diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc
index 01aa4e4..aac3bbd 100644
--- a/iocore/net/UnixNetAccept.cc
+++ b/iocore/net/UnixNetAccept.cc
@@ -38,6 +38,29 @@ safe_delay(int msec)
socketManager.poll(nullptr, 0, msec);
}
+static int
+drain_throttled_accepts(NetAccept *na)
+{
+ struct pollfd afd;
+ Connection con[THROTTLE_AT_ONCE];
+
+ afd.fd = na->server.fd;
+ afd.events = POLLIN;
+
+ // Try to close at most THROTTLE_AT_ONCE accept requests
+ // Stop if there is nothing waiting
+ int n = 0;
+ for (; n < THROTTLE_AT_ONCE && socketManager.poll(&afd, 1, 0) > 0; n++) {
+ int res = 0;
+ if ((res = na->server.accept(&con[n])) < 0) {
+ return res;
+ }
+ con[n].close();
+ }
+ // Return the number of accept cases we closed
+ return n;
+}
+
//
// General case network connection accept code
//
@@ -253,7 +276,19 @@ NetAccept::do_blocking_accept(EThread *t)
do {
ink_hrtime now = Thread::get_hrtime();
+ // Throttle accepts
+ while (!opt.backdoor && check_net_throttle(ACCEPT)) {
+ check_throttle_warning(ACCEPT);
+ int num_throttled = drain_throttled_accepts(this);
+ if (num_throttled < 0) {
+ goto Lerror;
+ }
+ NET_SUM_DYN_STAT(net_connections_throttled_in_stat, num_throttled);
+ now = Thread::get_hrtime();
+ }
+
if ((res = server.accept(&con)) < 0) {
+ Lerror:
int seriousness = accept_error_seriousness(res);
if (seriousness >= 0) { // not so bad
if (!seriousness) { // bad enough to warn about
@@ -270,20 +305,6 @@ NetAccept::do_blocking_accept(EThread *t)
return -1;
}
- // Throttle accepts
- if (!opt.backdoor && (check_net_throttle(ACCEPT, now) || net_memory_throttle)) {
- Debug("net_accept", "Too many connections or too much memory used, throttling");
- check_throttle_warning();
- con.close();
- continue;
- }
-
- // The con.fd may exceed the limitation of check_net_throttle() because we do blocking accept here.
- if (check_emergency_throttle(con)) {
- con.close();
- return 0;
- }
-
// Use 'nullptr' to Bypass thread allocator
vc = (UnixNetVConnection *)this->getNetProcessor()->allocate_vc(nullptr);
if (unlikely(!vc || shutdown_event_system == true)) {
@@ -374,7 +395,7 @@ NetAccept::acceptFastEvent(int event, void *ep)
int loop = accept_till_done;
do {
- if (!opt.backdoor && check_net_throttle(ACCEPT, Thread::get_hrtime())) {
+ if (!opt.backdoor && check_net_throttle(ACCEPT)) {
ifd = NO_FD;
return EVENT_CONT;
}
diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc
index b96b8e0..a7310b1 100644
--- a/iocore/net/UnixNetVConnection.cc
+++ b/iocore/net/UnixNetVConnection.cc
@@ -1234,9 +1234,10 @@ UnixNetVConnection::connectUp(EThread *t, int fd)
int res;
thread = t;
- if (check_net_throttle(CONNECT, submit_time)) {
- check_throttle_warning();
+ if (check_net_throttle(CONNECT)) {
+ check_throttle_warning(CONNECT);
res = -ENET_THROTTLING;
+ NET_INCREMENT_DYN_STAT(net_connections_throttled_out_stat);
goto fail;
}
@@ -1274,13 +1275,6 @@ UnixNetVConnection::connectUp(EThread *t, int fd)
con.is_bound = true;
}
- if (check_emergency_throttle(con)) {
- // Set errno force to EMFILE (reached limit for open file descriptors)
- errno = EMFILE;
- res = -errno;
- goto fail;
- }
-
// Must connect after EventIO::Start() to avoid a race condition
// when edge triggering is used.
if ((res = get_NetHandler(t)->startIO(this)) < 0) {
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index 28d8be4..6ffbdf5 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -1795,6 +1795,9 @@ HttpSM::state_http_server_open(int event, void *data)
}
t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; // part of the problem, clear it.
terminate_sm = true;
+ } else if (ENET_THROTTLING == t_state.current.server->connect_result) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_connections_throttled_stat);
+ send_origin_throttled_response();
} else {
call_transact_and_set_next_state(HttpTransact::HandleResponse);
}
diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h
index 1b44e9b..d98d3a4 100644
--- a/proxy/http/HttpSM.h
+++ b/proxy/http/HttpSM.h
@@ -181,7 +181,6 @@ class PostDataBuffers
{
public:
PostDataBuffers() { Debug("http_redirect", "[PostDataBuffers::PostDataBuffers]"); }
-
MIOBuffer *postdata_copy_buffer = nullptr;
IOBufferReader *postdata_copy_buffer_start = nullptr;
IOBufferReader *ua_buffer_reader = nullptr;
--
To stop receiving notification emails like this one, please contact
shinrich@apache.org.