You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2020/06/29 22:47:34 UTC
[trafficserver] branch 9.0.x updated: Metrics for origin close
(#6873)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/9.0.x by this push:
new 3e4b88e Metrics for origin close (#6873)
3e4b88e is described below
commit 3e4b88ed705417fbc702c0e3f481dd84d80679bd
Author: Susan Hinrichs <sh...@yahoo-inc.com>
AuthorDate: Mon Jun 29 08:40:09 2020 -0500
Metrics for origin close (#6873)
(cherry picked from commit 613576f1ba0252778a42acf22d2ff70d2dc10b9f)
---
.../monitoring/statistics/core/origin.en.rst | 59 ++++++++++++++++++++++
proxy/http/Http1ServerSession.cc | 1 +
proxy/http/HttpConfig.cc | 38 ++++++++++++++
proxy/http/HttpConfig.h | 19 +++++++
proxy/http/HttpSM.cc | 32 +++++++++++-
proxy/http/HttpSessionManager.cc | 1 +
proxy/http/HttpTunnel.cc | 1 +
7 files changed, 150 insertions(+), 1 deletion(-)
diff --git a/doc/admin-guide/monitoring/statistics/core/origin.en.rst b/doc/admin-guide/monitoring/statistics/core/origin.en.rst
index 2b325c4..528d4e1 100644
--- a/doc/admin-guide/monitoring/statistics/core/origin.en.rst
+++ b/doc/admin-guide/monitoring/statistics/core/origin.en.rst
@@ -78,4 +78,63 @@ Origin Server
:type: derivative
:units: bytes
+.. ts:stat:: global proxy.process.http.origin_shutdown.pool_lock_contention integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.migration_failure integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_eos integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_transform_read integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_sharing integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_keep_alive integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_repsonse integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_request integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_modified integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_misc integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.cleanup_entry integer
+ :type counter
+ :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_abort integer
+ :type counter
+ :units bytes
diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/Http1ServerSession.cc
index 88c8f55..2adaf28 100644
--- a/proxy/http/Http1ServerSession.cc
+++ b/proxy/http/Http1ServerSession.cc
@@ -196,6 +196,7 @@ Http1ServerSession::release()
// due to lock contention
// FIX: should retry instead of closing
this->do_io_close();
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_pool_lock_contention);
} else {
// The session was successfully put into the session
// manager and it will manage it
diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc
index 5877bc5..d3cc7b6 100644
--- a/proxy/http/HttpConfig.cc
+++ b/proxy/http/HttpConfig.cc
@@ -350,6 +350,44 @@ register_stat_callbacks()
RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.total_parent_marked_down_count", RECD_COUNTER, RECP_PERSISTENT,
(int)http_total_parent_marked_down_count, RecRawStatSyncCount);
+ // Stats to track causes of ATS initiated origin shutdowns
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.pool_lock_contention", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_pool_lock_contention, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.migration_failure", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_migration_failure, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_tunnel_server, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_no_keep_alive, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_eos", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_tunnel_server_eos, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_plugin_tunnel, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_detach", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_detach, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_client", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_tunnel_client, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_transform_read", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_transform_read, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_sharing", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_release_no_sharing, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_server", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_release_no_server, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_keep_alive", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_no_keep_alive, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_response", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_response, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_request", RECD_INT,
+ RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_request, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_modified", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_release_modified, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_misc", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_release_misc, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.cleanup_entry", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_cleanup_entry, RecRawStatSyncCount);
+ RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_abort", RECD_INT, RECP_NON_PERSISTENT,
+ (int)http_origin_shutdown_tunnel_abort, RecRawStatSyncCount);
+
// Upstream current connections stats
RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.current_parent_proxy_connections", RECD_INT, RECP_NON_PERSISTENT,
(int)http_current_parent_proxy_connections_stat, RecRawStatSyncSum);
diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h
index c7ba480..659e8e9 100644
--- a/proxy/http/HttpConfig.h
+++ b/proxy/http/HttpConfig.h
@@ -331,6 +331,25 @@ enum {
http_origin_connect_adjust_thread_stat,
http_cache_open_write_adjust_thread_stat,
+ http_origin_shutdown_pool_lock_contention,
+ http_origin_shutdown_migration_failure,
+ http_origin_shutdown_tunnel_server,
+ http_origin_shutdown_tunnel_server_no_keep_alive,
+ http_origin_shutdown_tunnel_server_eos,
+ http_origin_shutdown_tunnel_server_plugin_tunnel,
+ http_origin_shutdown_tunnel_server_detach,
+ http_origin_shutdown_tunnel_client,
+ http_origin_shutdown_tunnel_transform_read,
+ http_origin_shutdown_release_no_sharing,
+ http_origin_shutdown_release_no_server,
+ http_origin_shutdown_release_no_keep_alive,
+ http_origin_shutdown_release_invalid_response,
+ http_origin_shutdown_release_invalid_request,
+ http_origin_shutdown_release_modified,
+ http_origin_shutdown_release_misc,
+ http_origin_shutdown_cleanup_entry,
+ http_origin_shutdown_tunnel_abort,
+
http_stat_count
};
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index 775a6f9..c5914ec 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -229,6 +229,9 @@ HttpVCTable::cleanup_entry(HttpVCTableEntry *e)
break;
}
+ if (e->vc_type == HTTP_SERVER_VC) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_cleanup_entry);
+ }
e->vc->do_io_close();
e->vc = nullptr;
}
@@ -2940,6 +2943,13 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p)
plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL && t_state.txn_conf->keep_alive_enabled_out == 1) {
close_connection = false;
} else {
+ if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_no_keep_alive);
+ } else if (server_entry->eos == true) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_eos);
+ } else {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_plugin_tunnel);
+ }
close_connection = true;
}
@@ -2968,6 +2978,7 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p)
break;
}
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server);
close_connection = true;
ink_assert(p->vc_type == HT_HTTP_SERVER);
@@ -3036,7 +3047,8 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p)
p->read_success = true;
t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE;
t_state.current.server->abort = HttpTransact::DIDNOT_ABORT;
- close_connection = true;
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_detach);
+ close_connection = true;
break;
case VC_EVENT_READ_READY:
@@ -3899,6 +3911,7 @@ HttpSM::tunnel_handler_transform_read(int event, HttpTunnelProducer *p)
// transform hasn't detached yet. If it is still alive,
// don't close the transform vc
if (p->self_consumer->alive == false) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_transform_read);
p->vc->do_io_close();
}
p->handler_state = HTTP_SM_TRANSFORM_CLOSED;
@@ -5398,6 +5411,23 @@ HttpSM::release_server_session(bool serve_from_cache)
ua_txn->attach_server_session(server_session, false);
}
} else {
+ if (TS_SERVER_SESSION_SHARING_MATCH_NONE == t_state.txn_conf->server_session_sharing_match) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_sharing);
+ } else if (t_state.current.server == nullptr) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_server);
+ } else if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_keep_alive);
+ } else if (!t_state.hdr_info.server_response.valid()) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_response);
+ } else if (!t_state.hdr_info.server_request.valid()) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_request);
+ } else if (t_state.hdr_info.server_response.status_get() != HTTP_STATUS_NOT_MODIFIED &&
+ (t_state.hdr_info.server_request.method_get_wksidx() != HTTP_WKSIDX_HEAD ||
+ t_state.www_auth_content == HttpTransact::CACHE_AUTH_NONE)) {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_modified);
+ } else {
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_misc);
+ }
server_session->do_io_close();
}
diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc
index a2a1264..ac2fed4 100644
--- a/proxy/http/HttpSessionManager.cc
+++ b/proxy/http/HttpSessionManager.cc
@@ -401,6 +401,7 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad
ink_assert(new_vc == nullptr || new_vc->nh != nullptr);
if (!new_vc) {
// Close out to_return, we were't able to get a connection
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_migration_failure);
to_return->do_io_close();
to_return = nullptr;
retval = HSM_NOT_FOUND;
diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc
index 1bfc10b..0588acd 100644
--- a/proxy/http/HttpTunnel.cc
+++ b/proxy/http/HttpTunnel.cc
@@ -1419,6 +1419,7 @@ HttpTunnel::chain_abort_all(HttpTunnelProducer *p)
}
p->read_vio = nullptr;
p->vc->do_io_close(EHTTP_ERROR);
+ HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_abort);
update_stats_after_abort(p->vc_type);
}
}