You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by bc...@apache.org on 2016/10/15 23:42:48 UTC
[trafficserver] branch master updated: TS-4909: Throttling based on
resident memory
This is an automated email from the ASF dual-hosted git repository.
bcall pushed a commit to branch master
in repository https://git-dual.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new cbd5cc1 TS-4909: Throttling based on resident memory
cbd5cc1 is described below
commit cbd5cc1f9229235df44f8fcf1262eb491a48f2de
Author: Bryan Call <bc...@apache.org>
AuthorDate: Wed Sep 28 21:17:43 2016 -0700
TS-4909: Throttling based on resident memory
---
iocore/net/P_UnixNet.h | 2 +-
iocore/net/P_UnixNetProcessor.h | 1 -
iocore/net/UnixNet.cc | 1 +
iocore/net/UnixNetAccept.cc | 54 ++++++-----------------------------------
mgmt/RecordsConfig.cc | 2 ++
proxy/Main.cc | 50 ++++++++++++++++++++++++++++++++++++++
6 files changed, 62 insertions(+), 48 deletions(-)
diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h
index 781ae93..e12f3fd 100644
--- a/iocore/net/P_UnixNet.h
+++ b/iocore/net/P_UnixNet.h
@@ -124,6 +124,7 @@ extern ink_hrtime last_throttle_warning;
extern ink_hrtime last_shedding_warning;
extern ink_hrtime emergency_throttle_time;
extern int net_connections_throttle;
+extern bool net_memory_throttle;
extern int fds_throttle;
extern int fds_limit;
extern ink_hrtime last_transient_accept_error;
@@ -141,7 +142,6 @@ extern int http_accept_port_number;
#define TRANSIENT_ACCEPT_ERROR_MESSAGE_EVERY HRTIME_HOURS(24)
// also the 'throttle connect headroom'
-#define THROTTLE_AT_ONCE 5
#define EMERGENCY_THROTTLE 16
#define HYPER_EMERGENCY_THROTTLE 6
diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h
index 8f30d5d..a3285be 100644
--- a/iocore/net/P_UnixNetProcessor.h
+++ b/iocore/net/P_UnixNetProcessor.h
@@ -45,7 +45,6 @@ public:
virtual int start(int number_of_net_threads, size_t stacksize);
- char *throttle_error_message;
Event *accept_thread_event;
// offsets for per thread data structures
diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc
index b3a5628..d4a082d 100644
--- a/iocore/net/UnixNet.cc
+++ b/iocore/net/UnixNet.cc
@@ -27,6 +27,7 @@ ink_hrtime last_throttle_warning;
ink_hrtime last_shedding_warning;
ink_hrtime emergency_throttle_time;
int net_connections_throttle;
+bool net_memory_throttle = false;
int fds_throttle;
int fds_limit = 8000;
ink_hrtime last_transient_accept_error;
diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc
index 7d35ad0..b049275 100644
--- a/iocore/net/UnixNetAccept.cc
+++ b/iocore/net/UnixNetAccept.cc
@@ -39,39 +39,6 @@ safe_delay(int msec)
}
//
-// Send the throttling message to up to THROTTLE_AT_ONCE connections,
-// delaying to let some of the current connections complete
-//
-static int
-send_throttle_message(NetAccept *na)
-{
- struct pollfd afd;
- Connection con[100];
- char dummy_read_request[4096];
-
- afd.fd = na->server.fd;
- afd.events = POLLIN;
-
- int n = 0;
- while (check_net_throttle(ACCEPT, Thread::get_hrtime()) && n < THROTTLE_AT_ONCE - 1 && (socketManager.poll(&afd, 1, 0) > 0)) {
- int res = 0;
- if ((res = na->server.accept(&con[n])) < 0)
- return res;
- n++;
- }
- safe_delay(net_throttle_delay / 2);
- int i = 0;
- for (i = 0; i < n; i++) {
- socketManager.read(con[i].fd, dummy_read_request, 4096);
- socketManager.write(con[i].fd, unix_netProcessor.throttle_error_message, strlen(unix_netProcessor.throttle_error_message));
- }
- safe_delay(net_throttle_delay / 2);
- for (i = 0; i < n; i++)
- con[i].close();
- return 0;
-}
-
-//
// General case network connection accept code
//
int
@@ -256,20 +223,7 @@ NetAccept::do_blocking_accept(EThread *t)
do {
ink_hrtime now = Thread::get_hrtime();
- // Throttle accepts
-
- while (!opt.backdoor && check_net_throttle(ACCEPT, now)) {
- check_throttle_warning();
- if (!unix_netProcessor.throttle_error_message) {
- safe_delay(net_throttle_delay);
- } else if (send_throttle_message(this) < 0) {
- goto Lerror;
- }
- now = Thread::get_hrtime();
- }
-
if ((res = server.accept(&con)) < 0) {
- Lerror:
int seriousness = accept_error_seriousness(res);
if (seriousness >= 0) { // not so bad
if (!seriousness) // bad enough to warn about
@@ -285,6 +239,14 @@ NetAccept::do_blocking_accept(EThread *t)
return -1;
}
+ // Throttle accepts
+ if (!backdoor && (check_net_throttle(ACCEPT, now) || net_memory_throttle)) {
+ Debug("net_accept", "Too many connections or too much memory used, throttling");
+ check_throttle_warning();
+ con.close();
+ continue;
+ }
+
// The con.fd may exceed the limitation of check_net_throttle() because we do blocking accept here.
if (check_emergency_throttle(con)) {
// The `con' could be closed if there is hyper emergency
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index 6778bbd..f6183c3 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -110,6 +110,8 @@ static const RecordElement RecordsConfig[] =
//# 0 = disable
{RECT_CONFIG, "proxy.config.res_track_memory", RECD_INT, "0", RECU_RESTART_TS, RR_REQUIRED, RECC_INT, "[0-2]", RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.memory.max_usage", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL}
+ ,
//##############################################################################
//# Traffic Server system settings
//##############################################################################
diff --git a/proxy/Main.cc b/proxy/Main.cc
index 7334c55..416fc45 100644
--- a/proxy/Main.cc
+++ b/proxy/Main.cc
@@ -346,6 +346,55 @@ public:
}
};
+class MemoryLimit : public Continuation
+{
+public:
+ MemoryLimit() : Continuation(new_ProxyMutex()), _memory_limit(0) { SET_HANDLER(&MemoryLimit::periodic); }
+ ~MemoryLimit() { mutex = NULL; }
+ int
+ periodic(int event, Event *e)
+ {
+ if (event == EVENT_IMMEDIATE) {
+ // rescheduled from periodic to immediate event
+ // this is the indication to terminate
+ delete this;
+ return EVENT_DONE;
+ }
+ if (_memory_limit == 0) {
+ // first time it has been run
+ _memory_limit = REC_ConfigReadInteger("proxy.config.memory.max_usage");
+ _memory_limit = _memory_limit >> 10; // divide by 1024
+ }
+ if (_memory_limit > 0) {
+ if (getrusage(RUSAGE_SELF, &_usage) == 0) {
+ Debug("server", "memory usage - ru_maxrss: %ld memory limit: %" PRId64, _usage.ru_maxrss, _memory_limit);
+ if (_usage.ru_maxrss > _memory_limit) {
+ if (net_memory_throttle == false) {
+ net_memory_throttle = true;
+ Debug("server", "memory usage exceeded limit - ru_maxrss: %ld memory limit: %" PRId64, _usage.ru_maxrss, _memory_limit);
+ }
+ } else {
+ if (net_memory_throttle == true) {
+ net_memory_throttle = false;
+ Debug("server", "memory usage under limit - ru_maxrss: %ld memory limit: %" PRId64, _usage.ru_maxrss, _memory_limit);
+ }
+ }
+ }
+ } else {
+ // this feature has not be enabled
+ Debug("server", "limiting connections based on memory usage has been disabled");
+ e->cancel();
+ delete this;
+ return EVENT_DONE;
+ }
+ return EVENT_CONT;
+ }
+
+private:
+ int64_t _memory_limit;
+ struct rusage _usage;
+};
+
static int
init_memory_tracker(const char *config_var, RecDataT /* type ATS_UNUSED */, RecData data, void * /* cookie ATS_UNUSED */)
{
@@ -1730,6 +1779,7 @@ main(int /* argc ATS_UNUSED */, const char **argv)
eventProcessor.schedule_every(new SignalContinuation, HRTIME_MSECOND * 500, ET_CALL);
eventProcessor.schedule_every(new DiagsLogContinuation, HRTIME_SECOND, ET_TASK);
+ eventProcessor.schedule_every(new MemoryLimit, HRTIME_SECOND, ET_TASK);
REC_RegisterConfigUpdateFunc("proxy.config.dump_mem_info_frequency", init_memory_tracker, NULL);
init_memory_tracker(NULL, RECD_NULL, RecData(), NULL);
--
To stop receiving notification emails like this one, please contact
['"commits@trafficserver.apache.org" <co...@trafficserver.apache.org>'].