You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2018/03/08 18:06:57 UTC
[trafficserver] branch 7.1.x updated: Make throttling feature more
useful.
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 7.1.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/7.1.x by this push:
new bd72b1e Make throttling feature more useful.
bd72b1e is described below
commit bd72b1e262dc5bae6849e8091590e1e785d4aaa9
Author: Susan Hinrichs <sh...@apache.org>
AuthorDate: Wed Feb 7 01:04:22 2018 +0000
Make throttling feature more useful.
(cherry picked from commit 4bfaf364529d49f576e2f3d27b3388364ca4606e)
Conflicts:
iocore/net/Net.cc
iocore/net/P_Net.h
iocore/net/P_UnixNet.h
iocore/net/UnixNetAccept.cc
iocore/net/UnixNetVConnection.cc
---
iocore/net/Net.cc | 5 ++++
iocore/net/P_Net.h | 2 ++
iocore/net/P_UnixNet.h | 47 ++++++-----------------------------
iocore/net/UnixNet.cc | 1 -
iocore/net/UnixNetAccept.cc | 53 +++++++++++++++++++++++++++-------------
iocore/net/UnixNetVConnection.cc | 22 ++++-------------
proxy/http/HttpSM.cc | 3 +++
7 files changed, 58 insertions(+), 75 deletions(-)
diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc
index 6e6c35e..0984df3 100644
--- a/iocore/net/Net.cc
+++ b/iocore/net/Net.cc
@@ -108,6 +108,11 @@ register_net_stats()
NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_total_stat);
NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat);
NET_CLEAR_DYN_STAT(default_inactivity_timeout_stat);
+
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_in", RECD_INT, RECP_PERSISTENT,
+ (int)net_connections_throttled_in_stat, RecRawStatSyncSum);
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT,
+ (int)net_connections_throttled_out_stat, RecRawStatSyncSum);
}
void
diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h
index c2c2fd9..d9dde8d 100644
--- a/iocore/net/P_Net.h
+++ b/iocore/net/P_Net.h
@@ -55,6 +55,8 @@ enum Net_Stats {
default_inactivity_timeout_stat,
net_fastopen_attempts_stat,
net_fastopen_successes_stat,
+ net_connections_throttled_in_stat,
+ net_connections_throttled_out_stat,
Net_Stat_Count
};
diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h
index f201bec..c0d57b3 100644
--- a/iocore/net/P_UnixNet.h
+++ b/iocore/net/P_UnixNet.h
@@ -144,6 +144,7 @@ extern int http_accept_port_number;
// also the 'throttle connect headroom'
#define EMERGENCY_THROTTLE 16
+#define THROTTLE_AT_ONCE 5
#define HYPER_EMERGENCY_THROTTLE 6
#define NET_THROTTLE_ACCEPT_HEADROOM 1.1 // 10%
@@ -270,61 +271,27 @@ check_shedding_warning()
}
TS_INLINE bool
-emergency_throttle(ink_hrtime now)
-{
- return (bool)(emergency_throttle_time > now);
-}
-
-TS_INLINE bool
-check_net_throttle(ThrottleType t, ink_hrtime now)
+check_net_throttle(ThrottleType t)
{
int connections = net_connections_to_throttle(t);
if (connections >= net_connections_throttle)
return true;
- if (emergency_throttle(now))
- return true;
-
return false;
}
TS_INLINE void
-check_throttle_warning()
+check_throttle_warning(ThrottleType type)
{
ink_hrtime t = Thread::get_hrtime();
if (t - last_throttle_warning > NET_THROTTLE_MESSAGE_EVERY) {
last_throttle_warning = t;
- RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR, "too many connections, throttling");
- }
-}
-
-//
-// Emergency throttle when we are close to exhausting file descriptors.
-// Block all accepts or connects for N seconds where N
-// is the amount into the emergency fd stack squared
-// (e.g. on the last file descriptor we have 14 * 14 = 196 seconds
-// of emergency throttle).
-//
-// Hyper Emergency throttle when we are very close to exhausting file
-// descriptors. Close the connection immediately, the upper levels
-// will recover.
-//
-TS_INLINE bool
-check_emergency_throttle(Connection &con)
-{
- int fd = con.fd;
- int emergency = fds_limit - EMERGENCY_THROTTLE;
- if (fd > emergency) {
- int over = fd - emergency;
- emergency_throttle_time = Thread::get_hrtime() + (over * over) * HRTIME_SECOND;
- RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR, "too many open file descriptors, emergency throttling");
- int hyper_emergency = fds_limit - HYPER_EMERGENCY_THROTTLE;
- if (fd > hyper_emergency)
- con.close();
- return true;
+ int connections = net_connections_to_throttle(type);
+ RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR,
+ "too many connections, throttling. connection_type=%s, current_connections=%d, net_connections_throttle=%d",
+ type == ACCEPT ? "ACCEPT" : "CONNECT", connections, net_connections_throttle);
}
- return false;
}
TS_INLINE int
diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc
index 6c793ed..06da414 100644
--- a/iocore/net/UnixNet.cc
+++ b/iocore/net/UnixNet.cc
@@ -25,7 +25,6 @@
ink_hrtime last_throttle_warning;
ink_hrtime last_shedding_warning;
-ink_hrtime emergency_throttle_time;
int net_connections_throttle;
bool net_memory_throttle = false;
int fds_throttle;
diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc
index f0bc3c8..77408d4 100644
--- a/iocore/net/UnixNetAccept.cc
+++ b/iocore/net/UnixNetAccept.cc
@@ -39,6 +39,29 @@ safe_delay(int msec)
socketManager.poll(nullptr, 0, msec);
}
+static int
+drain_throttled_accepts(NetAccept *na)
+{
+ struct pollfd afd;
+ Connection con[THROTTLE_AT_ONCE];
+
+ afd.fd = na->server.fd;
+ afd.events = POLLIN;
+
+ // Try to close at most THROTTLE_AT_ONCE accept requests
+ // Stop if there is nothing waiting
+ int n = 0;
+ for (; n < THROTTLE_AT_ONCE && socketManager.poll(&afd, 1, 0) > 0; n++) {
+ int res = 0;
+ if ((res = na->server.accept(&con[n])) < 0) {
+ return res;
+ }
+ con[n].close();
+ }
+ // Return the number of accept cases we closed
+ return n;
+}
+
//
// General case network connection accept code
//
@@ -230,7 +253,19 @@ NetAccept::do_blocking_accept(EThread *t)
do {
ink_hrtime now = Thread::get_hrtime();
+ // Throttle accepts
+ while (!opt.backdoor && check_net_throttle(ACCEPT)) {
+ check_throttle_warning(ACCEPT);
+ int num_throttled = drain_throttled_accepts(this);
+ if (num_throttled < 0) {
+ goto Lerror;
+ }
+ NET_SUM_DYN_STAT(net_connections_throttled_in_stat, num_throttled);
+ now = Thread::get_hrtime();
+ }
+
if ((res = server.accept(&con)) < 0) {
+ Lerror:
int seriousness = accept_error_seriousness(res);
if (seriousness >= 0) { // not so bad
if (!seriousness) // bad enough to warn about
@@ -246,22 +281,6 @@ NetAccept::do_blocking_accept(EThread *t)
return -1;
}
- // Throttle accepts
- if (!opt.backdoor && (check_net_throttle(ACCEPT, now) || net_memory_throttle)) {
- Debug("net_accept", "Too many connections or too much memory used, throttling");
- check_throttle_warning();
- con.close();
- continue;
- }
-
- // The con.fd may exceed the limitation of check_net_throttle() because we do blocking accept here.
- if (check_emergency_throttle(con)) {
- // The `con' could be closed if there is hyper emergency
- if (con.fd == NO_FD) {
- return 0;
- }
- }
-
// Use 'nullptr' to Bypass thread allocator
vc = (UnixNetVConnection *)this->getNetProcessor()->allocate_vc(nullptr);
if (unlikely(!vc || shutdown_event_system == true)) {
@@ -346,7 +365,7 @@ NetAccept::acceptFastEvent(int event, void *ep)
int loop = accept_till_done;
do {
- if (!opt.backdoor && check_net_throttle(ACCEPT, Thread::get_hrtime())) {
+ if (!opt.backdoor && check_net_throttle(ACCEPT)) {
ifd = NO_FD;
return EVENT_CONT;
}
diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc
index 5e246c1..2f1014a 100644
--- a/iocore/net/UnixNetVConnection.cc
+++ b/iocore/net/UnixNetVConnection.cc
@@ -1303,11 +1303,11 @@ UnixNetVConnection::connectUp(EThread *t, int fd)
int res;
thread = t;
- if (check_net_throttle(CONNECT, submit_time)) {
- check_throttle_warning();
- action_.continuation->handleEvent(NET_EVENT_OPEN_FAILED, (void *)-ENET_THROTTLING);
- free(t);
- return CONNECT_FAILURE;
+ if (check_net_throttle(CONNECT)) {
+ check_throttle_warning(CONNECT);
+ res = -ENET_THROTTLING;
+ NET_INCREMENT_DYN_STAT(net_connections_throttled_out_stat);
+ goto fail;
}
// Force family to agree with remote (server) address.
@@ -1344,18 +1344,6 @@ UnixNetVConnection::connectUp(EThread *t, int fd)
con.is_bound = true;
}
- if (check_emergency_throttle(con)) {
- // The `con' could be closed if there is hyper emergency
- if (con.fd == NO_FD) {
- // We need to decrement the stat because close_UnixNetVConnection only decrements with a valid connection descriptor.
- NET_SUM_GLOBAL_DYN_STAT(net_connections_currently_open_stat, -1);
- // Set errno force to EMFILE (reached limit for open file descriptors)
- errno = EMFILE;
- res = -errno;
- goto fail;
- }
- }
-
// Must connect after EventIO::Start() to avoid a race condition
// when edge triggering is used.
if (ep.start(get_PollDescriptor(t), this, EVENTIO_READ | EVENTIO_WRITE) < 0) {
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index 6bb2751..7658624 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -1806,6 +1806,9 @@ HttpSM::state_http_server_open(int event, void *data)
}
t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; // part of the problem, clear it.
terminate_sm = true;
+ } else if (ENET_THROTTLING == t_state.current.server->connect_result) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_connections_throttled_stat);
+ send_origin_throttled_response();
} else {
call_transact_and_set_next_state(HttpTransact::HandleResponse);
}
--
To stop receiving notification emails like this one, please contact
zwoop@apache.org.