You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by bc...@apache.org on 2015/01/21 19:33:52 UTC
trafficserver git commit: TS-153: Dynamic keep-alive timeouts
Repository: trafficserver
Updated Branches:
refs/heads/master 86bda3532 -> 5e91acd24
TS-153: Dynamic keep-alive timeouts
1. Changed the limit on the number of connections to include all
incoming connections instead of just ones that are keep-alive. This will
keep the number of incoming connections per thread more consistent.
However, it will only close keep-alive connections.
2. Change the double link list to a queue.
3. When adding to the queue if the connection is already in the queue
remove it and then add it to the end the queue.
4. Properly close the connection by mimicking what inactivity cop does
to close the connection.
5. Add stats to determine the average KA timeout since it is now dynamic.
6. Added support for spdy connections.
Config option is now:
proxy.config.net.connections.threshold_shed_idle_in
Stats added are:
proxy.process.net.dynamic_keep_alive_timeout_in_total
proxy.process.net.dynamic_keep_alive_timeout_in_count
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/5e91acd2
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/5e91acd2
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/5e91acd2
Branch: refs/heads/master
Commit: 5e91acd24e67da73baa81ded14878e32273f1e20
Parents: 86bda35
Author: Bryan Call <bc...@apache.org>
Authored: Wed Jan 21 10:25:35 2015 -0800
Committer: Bryan Call <bc...@apache.org>
Committed: Wed Jan 21 10:25:35 2015 -0800
----------------------------------------------------------------------
iocore/net/Net.cc | 10 +++
iocore/net/P_Net.h | 2 +
iocore/net/P_UnixNet.h | 2 +-
iocore/net/UnixConnection.cc | 7 +-
iocore/net/UnixNet.cc | 122 +++++++++++++++++++++++++++--------
mgmt/RecordsConfig.cc | 8 +--
proxy/spdy/SpdyCallbacks.cc | 1 +
proxy/spdy/SpdyClientSession.cc | 2 +-
proxy/spdy/SpdyClientSession.h | 3 +
9 files changed, 121 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/iocore/net/Net.cc
----------------------------------------------------------------------
diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc
index 11c0c48..e97070c 100644
--- a/iocore/net/Net.cc
+++ b/iocore/net/Net.cc
@@ -111,6 +111,16 @@ register_net_stats()
RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.inactivity_cop_lock_acquire_failure",
RECD_INT, RECP_PERSISTENT, (int) inactivity_cop_lock_acquire_failure_stat,
RecRawStatSyncSum);
+
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.dynamic_keep_alive_timeout_in_total",
+ RECD_INT, RECP_NON_PERSISTENT, (int) keep_alive_lru_timeout_total_stat,
+ RecRawStatSyncSum);
+ NET_CLEAR_DYN_STAT(keep_alive_lru_timeout_total_stat);
+
+ RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.dynamic_keep_alive_timeout_in_count",
+ RECD_INT, RECP_NON_PERSISTENT, (int) keep_alive_lru_timeout_count_stat,
+ RecRawStatSyncSum);
+ NET_CLEAR_DYN_STAT(keep_alive_lru_timeout_count_stat);
}
void
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/iocore/net/P_Net.h
----------------------------------------------------------------------
diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h
index 97a6184..8dfa309 100644
--- a/iocore/net/P_Net.h
+++ b/iocore/net/P_Net.h
@@ -51,6 +51,8 @@ enum Net_Stats
socks_connections_unsuccessful_stat,
socks_connections_currently_open_stat,
inactivity_cop_lock_acquire_failure_stat,
+ keep_alive_lru_timeout_total_stat,
+ keep_alive_lru_timeout_count_stat,
Net_Stat_Count
};
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/iocore/net/P_UnixNet.h
----------------------------------------------------------------------
diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h
index 2f67515..7b8228a 100644
--- a/iocore/net/P_UnixNet.h
+++ b/iocore/net/P_UnixNet.h
@@ -192,7 +192,7 @@ public:
DList(UnixNetVConnection, cop_link) cop_list;
ASLLM(UnixNetVConnection, NetState, read, enable_link) read_enable_list;
ASLLM(UnixNetVConnection, NetState, write, enable_link) write_enable_list;
- DList(UnixNetVConnection, keep_alive_link) keep_alive_list;
+ Que(UnixNetVConnection, keep_alive_link) keep_alive_list;
uint32_t keep_alive_lru_size;
time_t sec;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/iocore/net/UnixConnection.cc
----------------------------------------------------------------------
diff --git a/iocore/net/UnixConnection.cc b/iocore/net/UnixConnection.cc
index d8daf45..ebe2791 100644
--- a/iocore/net/UnixConnection.cc
+++ b/iocore/net/UnixConnection.cc
@@ -394,8 +394,11 @@ void
UnixNetVConnection::add_to_keep_alive_lru()
{
Debug("socket", "UnixNetVConnection::add_to_keep_alive_lru NetVC=%p", this);
- if (! nh->keep_alive_list.in(this)) {
- nh->keep_alive_list.push(this);
+ if (nh->keep_alive_list.in(this)) {
+ nh->keep_alive_list.remove(this);
+ nh->keep_alive_list.enqueue(this);
+ } else {
+ nh->keep_alive_list.enqueue(this);
++nh->keep_alive_lru_size;
}
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/iocore/net/UnixNet.cc
----------------------------------------------------------------------
diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc
index fe9303d..c6c1ace 100644
--- a/iocore/net/UnixNet.cc
+++ b/iocore/net/UnixNet.cc
@@ -40,7 +40,7 @@ extern "C" void fd_reify(struct ev_loop *);
// loops through the list of NetVCs and calls the timeouts
class InactivityCop : public Continuation {
public:
- InactivityCop(ProxyMutex *m):Continuation(m), default_inactivity_timeout(0) {
+ InactivityCop(ProxyMutex *m):Continuation(m), default_inactivity_timeout(0), total_connections(0) {
SET_HANDLER(&InactivityCop::check_inactivity);
REC_ReadConfigInteger(default_inactivity_timeout, "proxy.config.net.default_inactivity_timeout");
Debug("inactivity_cop", "default inactivity timeout is set to: %d", default_inactivity_timeout);
@@ -48,13 +48,16 @@ public:
int check_inactivity(int event, Event *e) {
(void) event;
ink_hrtime now = ink_get_hrtime();
- NetHandler *nh = get_NetHandler(this_ethread());
+ NetHandler &nh = *get_NetHandler(this_ethread());
+ total_connections = 0;
// Copy the list and use pop() to catch any closes caused by callbacks.
- forl_LL(UnixNetVConnection, vc, nh->open_list) {
- if (vc->thread == this_ethread())
- nh->cop_list.push(vc);
+ forl_LL(UnixNetVConnection, vc, nh.open_list) {
+ if (vc->thread == this_ethread()) {
+ ++total_connections;
+ nh.cop_list.push(vc);
+ }
}
- while (UnixNetVConnection *vc = nh->cop_list.pop()) {
+ while (UnixNetVConnection *vc = nh.cop_list.pop()) {
// If we cannot get the lock don't stop just keep cleaning
MUTEX_TRY_LOCK(lock, vc->mutex, this_ethread());
if (!lock.is_locked()) {
@@ -69,41 +72,104 @@ public:
// set a default inactivity timeout if one is not set
if (vc->next_inactivity_timeout_at == 0 && default_inactivity_timeout > 0) {
- Debug("inactivity_cop", "vc: %p inactivity timeout not set, setting a default of %d", vc, default_inactivity_timeout);
+ Debug("inactivity_cop", "vc: %p inactivity timeout not set, setting a default of %d", vc,
+ default_inactivity_timeout);
vc->set_inactivity_timeout(HRTIME_SECONDS(default_inactivity_timeout));
} else {
- Debug("inactivity_cop_verbose", "vc: %p timeout at: %" PRId64 " timeout in: %" PRId64, vc, ink_hrtime_to_sec(vc->next_inactivity_timeout_at),
- ink_hrtime_to_sec(vc->inactivity_timeout_in));
+ Debug("inactivity_cop_verbose", "vc: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, vc,
+ now, ink_hrtime_to_sec(vc->next_inactivity_timeout_at), ink_hrtime_to_sec(vc->inactivity_timeout_in));
}
- if (vc->next_inactivity_timeout_at && vc->next_inactivity_timeout_at < now)
+ if (vc->next_inactivity_timeout_at && vc->next_inactivity_timeout_at < now) {
+ if (nh.keep_alive_list.in(vc)) {
+ // only stat if the connection is in keep-alive, there can be other inactivity timeouts
+ ink_hrtime diff = (now - (vc->next_inactivity_timeout_at - vc->inactivity_timeout_in)) / HRTIME_SECOND;
+ NET_SUM_DYN_STAT(keep_alive_lru_timeout_total_stat, diff);
+ NET_INCREMENT_DYN_STAT(keep_alive_lru_timeout_count_stat);
+ }
+ Debug("inactivity_cop_verbose", "vc: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, vc,
+ now, vc->next_inactivity_timeout_at, vc->inactivity_timeout_in);
vc->handleEvent(EVENT_IMMEDIATE, e);
+ }
}
// Keep-alive LRU for incoming connections
- int32_t max_keep_alive = 0;
- REC_ReadConfigInt32(max_keep_alive, "proxy.config.http.client_max_keep_alive_connections");
- if (max_keep_alive > 0) {
- const int event_threads = eventProcessor.n_threads_for_type[ET_NET];
- const int ssl_threads = (ET_NET == SSLNetProcessor::ET_SSL) ? 0 : eventProcessor.n_threads_for_type[SSLNetProcessor::ET_SSL];
-
- max_keep_alive = max_keep_alive / (event_threads + ssl_threads);
- Debug("inactivity_cop_verbose", "max_keep_alive: %d lru size: %d net threads: %d ssl threads: %d net type: %d "
- "ssl type: %d", max_keep_alive, nh->keep_alive_lru_size, event_threads, ssl_threads, ET_NET,
- SSLNetProcessor::ET_SSL);
-
- while (nh->keep_alive_lru_size > (uint32_t)max_keep_alive) {
- UnixNetVConnection *vc = nh->keep_alive_list.pop();
- Debug("inactivity_cop", "removing keep-alives from the lru NetVC=%p size: %u", vc, nh->keep_alive_lru_size);
- --(nh->keep_alive_lru_size);
- close_UnixNetVConnection(vc, e->ethread);
- }
- }
+ keep_alive_lru(nh, now, e);
+
return 0;
}
private:
+ void keep_alive_lru(NetHandler &nh, ink_hrtime now, Event *e);
int default_inactivity_timeout; // only used when one is not set for some bad reason
+ int total_connections;
};
+
+void InactivityCop::keep_alive_lru(NetHandler &nh, const ink_hrtime now, Event *e)
+{
+ // get the configuration value for the maximum incoming connections and check to see if it is set
+ int32_t connections_per_thread = 0;
+ REC_ReadConfigInt32(connections_per_thread, "proxy.config.net.connections.threshold_shed_idle_in");
+ if (connections_per_thread <= 0) {
+ Debug("inactivity_cop_dynamic", "dynamic keep-alive not configured");
+ return;
+ }
+
+ // figure out the number of threads and calculate the number of connections per thread
+ const int event_threads = eventProcessor.n_threads_for_type[ET_NET];
+ const int ssl_threads = (ET_NET == SSLNetProcessor::ET_SSL) ? 0 :
+ eventProcessor.n_threads_for_type[SSLNetProcessor::ET_SSL];
+ connections_per_thread = connections_per_thread / (event_threads + ssl_threads);
+
+ // calculate how many connections to close
+ int32_t to_process = total_connections - connections_per_thread;
+ if (to_process <= 0) {
+ return;
+ }
+ to_process = min((int32_t)nh.keep_alive_lru_size, to_process);
+
+ Debug("inactivity_cop_dynamic", "max cons: %d active: %d idle: %d process: %d net threads: %d ssl threads: %d"
+ "net type: %d ssl type: %d", connections_per_thread, total_connections - nh.keep_alive_lru_size,
+ nh.keep_alive_lru_size, to_process, event_threads, ssl_threads, ET_NET, SSLNetProcessor::ET_SSL);
+
+ // loop over the non-active connections and try to close them
+ UnixNetVConnection *vc = nh.keep_alive_list.head;
+ UnixNetVConnection *vc_next = NULL;
+ int closed = 0;
+ int handle_event = 0;
+ int total_idle_time = 0;
+ int total_idle_count = 0;
+ for (int32_t i = 0; i < to_process && vc != NULL; ++i, vc = vc_next) {
+ vc_next = vc->keep_alive_link.next;
+ if (vc->thread != this_ethread())
+ continue;
+ ink_hrtime diff = (now - (vc->next_inactivity_timeout_at - vc->inactivity_timeout_in)) / HRTIME_SECOND;
+ if (diff > 0) {
+ total_idle_time += diff;
+ ++total_idle_count;
+ NET_SUM_DYN_STAT(keep_alive_lru_timeout_total_stat, diff);
+ NET_INCREMENT_DYN_STAT(keep_alive_lru_timeout_count_stat);
+ }
+ Debug("inactivity_cop_dynamic", "closing connection NetVC=%p idle: %u now: %" PRId64 " at: %" PRId64
+ " in: %" PRId64 " diff: %" PRId64,
+ vc, nh.keep_alive_lru_size, ink_hrtime_to_sec(now), ink_hrtime_to_sec(vc->next_inactivity_timeout_at),
+ ink_hrtime_to_sec(vc->inactivity_timeout_in), diff);
+ if (vc->closed) {
+ vc->remove_from_keep_alive_lru();
+ close_UnixNetVConnection(vc, e->ethread);
+ ++closed;
+ } else {
+ vc->next_inactivity_timeout_at = now;
+ nh.keep_alive_list.head->handleEvent(EVENT_IMMEDIATE, e);
+ ++handle_event;
+ }
+ }
+
+ if (total_idle_count > 0) {
+ Debug("inactivity_cop_dynamic", "max cons: %d active: %d idle: %d already closed: %d, close event: %d"
+ "mean idle: %d\n", connections_per_thread, total_connections - nh.keep_alive_lru_size,
+ nh.keep_alive_lru_size, closed, handle_event, total_idle_time / total_idle_count);
+ }
+}
#endif
PollCont::PollCont(ProxyMutex *m, int pt):Continuation(m), net_handler(NULL), nextPollDescriptor(NULL), poll_timeout(pt) {
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/mgmt/RecordsConfig.cc
----------------------------------------------------------------------
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index fb5496c..fe94747 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -459,12 +459,12 @@ static const RecordElement RecordsConfig[] =
,
{RECT_CONFIG, "proxy.config.http.attach_server_session_to_client", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
,
- {RECT_CONFIG, "proxy.config.http.client_max_keep_alive_connections", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+ {RECT_CONFIG, "proxy.config.net.connections.threshold_shed_idle_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
,
- // ##########################
- // # HTTP referer filtering #
- // ##########################
+ // ###########################
+ // # HTTP referrer filtering #
+ // ###########################
{RECT_CONFIG, "proxy.config.http.referer_filter", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
,
{RECT_CONFIG, "proxy.config.http.referer_format_redirect", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/proxy/spdy/SpdyCallbacks.cc
----------------------------------------------------------------------
diff --git a/proxy/spdy/SpdyCallbacks.cc b/proxy/spdy/SpdyCallbacks.cc
index 64504af..138af6b 100644
--- a/proxy/spdy/SpdyCallbacks.cc
+++ b/proxy/spdy/SpdyCallbacks.cc
@@ -328,6 +328,7 @@ spdy_on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type,
req->init(sm, stream_id);
req->append_nv(frame->syn_stream.nv);
sm->req_map[stream_id] = req;
+ sm->vc->add_to_keep_alive_lru();
spdy_process_syn_stream_frame(sm, req);
break;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/proxy/spdy/SpdyClientSession.cc
----------------------------------------------------------------------
diff --git a/proxy/spdy/SpdyClientSession.cc b/proxy/spdy/SpdyClientSession.cc
index e003d0e..0788094 100644
--- a/proxy/spdy/SpdyClientSession.cc
+++ b/proxy/spdy/SpdyClientSession.cc
@@ -110,8 +110,8 @@ SpdyClientSession::init(NetVConnection * netvc, spdy::SessionVersion vers)
start_time = TShrtime();
this->vc->set_inactivity_timeout(HRTIME_SECONDS(spdy_accept_no_activity_timeout));
+ vc->add_to_keep_alive_lru();
SET_HANDLER(&SpdyClientSession::state_session_start);
-
}
void
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5e91acd2/proxy/spdy/SpdyClientSession.h
----------------------------------------------------------------------
diff --git a/proxy/spdy/SpdyClientSession.h b/proxy/spdy/SpdyClientSession.h
index e61c0ed..fc16d9b 100644
--- a/proxy/spdy/SpdyClientSession.h
+++ b/proxy/spdy/SpdyClientSession.h
@@ -146,6 +146,9 @@ public:
spdyRequestAllocator.free(req);
this->req_map.erase(streamId);
}
+ if (req_map.empty() == true) {
+ vc->add_to_keep_alive_lru();
+ }
}
private: