You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2023/08/28 19:50:46 UTC
[impala] branch master updated: IMPALA-10856: Show client hosts and connections in the web UI
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new bda3280cb IMPALA-10856: Show client hosts and connections in the web UI
bda3280cb is described below
commit bda3280cb45a689c27560292b9bbf28917cdc500
Author: ttttttz <24...@qq.com>
AuthorDate: Tue Aug 15 16:29:03 2023 +0800
IMPALA-10856: Show client hosts and connections in the web UI
This change adds two tables to display client hosts information
and connections information in the sessions Web UI. In the Client
Hosts table, include the following information items:
-- Hostname,
-- Total Connections,
-- Total Sessions,
-- Total Active Sessions,
-- Total Inactive Sessions,
-- Open Queries,
-- Total Queries
In the Connections table, include the following information items:
-- Connection ID,
-- Connection Type,
-- User,
-- Delegated User,
-- Network Address,
-- Session IDs
Change-Id: Ie89015b00e1b97a1836eeca205b2c80b32300227
Reviewed-on: http://gerrit.cloudera.org:8080/20358
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/rpc/thrift-server.cc | 7 ++
be/src/rpc/thrift-server.h | 5 ++
be/src/service/impala-http-handler.cc | 165 +++++++++++++++++++++++++++++++++-
be/src/service/impala-http-handler.h | 11 +++
be/src/service/impala-server.cc | 28 +++++-
be/src/service/impala-server.h | 9 +-
be/src/util/debug-util.cc | 12 +++
be/src/util/debug-util.h | 1 +
www/sessions.tmpl | 84 ++++++++++++++++-
9 files changed, 312 insertions(+), 10 deletions(-)
diff --git a/be/src/rpc/thrift-server.cc b/be/src/rpc/thrift-server.cc
index dec2671b2..7ee4bd7c5 100644
--- a/be/src/rpc/thrift-server.cc
+++ b/be/src/rpc/thrift-server.cc
@@ -434,4 +434,11 @@ void ThriftServer::StopForTesting() {
server_->stop();
if (started_) Join();
}
+
+void ThriftServer::GetConnectionContextList(ConnectionContextList* connection_contexts) {
+ lock_guard<mutex> l(connection_contexts_lock_);
+ for (const ConnectionContextSet::value_type& pair : connection_contexts_) {
+ connection_contexts->push_back(pair.second);
+ }
+}
}
diff --git a/be/src/rpc/thrift-server.h b/be/src/rpc/thrift-server.h
index f7ed7c10a..8b70cd270 100644
--- a/be/src/rpc/thrift-server.h
+++ b/be/src/rpc/thrift-server.h
@@ -178,6 +178,11 @@ class ThriftServer {
/// It is also only safe to reference the returned value during an RPC method.
static const ConnectionContext* GetThreadConnectionContext();
+ typedef std::vector<std::shared_ptr<const ConnectionContext>> ConnectionContextList;
+
+ /// Gets connection contexts for the thrift server.
+ void GetConnectionContextList(ConnectionContextList* connection_contexts);
+
private:
friend class ThriftServerBuilder;
friend class apache::thrift::server::TAcceptQueueServer;
diff --git a/be/src/service/impala-http-handler.cc b/be/src/service/impala-http-handler.cc
index cda45b343..5a4eca3d9 100644
--- a/be/src/service/impala-http-handler.cc
+++ b/be/src/service/impala-http-handler.cc
@@ -691,11 +691,24 @@ void ImpalaHttpHandler::QueryStateHandler(const Webserver::WebRequest& req,
void ImpalaHttpHandler::SessionsHandler(const Webserver::WebRequest& req,
Document* document) {
+ VLOG(1) << "Step1: Fill the sessions information into the document.";
+ FillSessionsInfo(document);
+
+ ThriftServer::ConnectionContextList connection_contexts;
+ server_->GetAllConnectionContexts(&connection_contexts);
+ VLOG(1) << "Step2: Fill the client hosts information into the document.";
+ FillClientHostsInfo(document, connection_contexts);
+
+ VLOG(1) << "Step3: Fill the connections information into the document.";
+ FillConnectionsInfo(document, connection_contexts);
+}
+
+void ImpalaHttpHandler::FillSessionsInfo(Document* document) {
lock_guard<mutex> l(server_->session_state_map_lock_);
Value sessions(kArrayType);
int num_active = 0;
- for (const ImpalaServer::SessionStateMap::value_type& session:
- server_->session_state_map_) {
+ for (const ImpalaServer::SessionStateMap::value_type& session :
+ server_->session_state_map_) {
shared_ptr<ImpalaServer::SessionState> state = session.second;
Value session_json(kObjectType);
Value type(PrintValue(state->session_type).c_str(), document->GetAllocator());
@@ -716,9 +729,9 @@ void ImpalaHttpHandler::SessionsHandler(const Webserver::WebRequest& req,
Value session_id(PrintId(session.first).c_str(), document->GetAllocator());
session_json.AddMember("session_id", session_id, document->GetAllocator());
- Value network_address(TNetworkAddressToString(state->network_address).c_str(),
+ Value connection_ids(PrintIdSet(state->connections, "\n").c_str(),
document->GetAllocator());
- session_json.AddMember("network_address", network_address, document->GetAllocator());
+ session_json.AddMember("connection_ids", connection_ids, document->GetAllocator());
Value default_db(state->database.c_str(), document->GetAllocator());
session_json.AddMember("default_database", default_db, document->GetAllocator());
@@ -753,6 +766,150 @@ void ImpalaHttpHandler::SessionsHandler(const Webserver::WebRequest& req,
document->GetAllocator());
}
+void ImpalaHttpHandler::FillClientHostsInfo(
+ Document* document, const ThriftServer::ConnectionContextList& connection_contexts) {
+ lock_guard<mutex> session_state_map_l(server_->session_state_map_lock_);
+ lock_guard<mutex> connection_to_sessions_map_l(
+ server_->connection_to_sessions_map_lock_);
+ Value client_hosts(kArrayType);
+ /// Map from a client hostname to the associated list of connections.
+ std::map<std::string, std::set<TUniqueId>> client_hostname_to_connections_map;
+ for (const ThriftServer::ConnectionContextList::value_type& connection_context :
+ connection_contexts) {
+ client_hostname_to_connections_map[connection_context->network_address.hostname]
+ .insert(connection_context->connection_id);
+ }
+ for (const auto& pair : client_hostname_to_connections_map) {
+ std::set<TUniqueId> client_hostname_sessions;
+ Value client_host_json(kObjectType);
+ int64_t total_connections = 0;
+ int64_t total_sessions = 0;
+ int64_t total_active_sessions = 0;
+ int64_t total_inactive_sessions = 0;
+ int64_t inflight_queries = 0;
+ int64_t total_queries = 0;
+ std::set<TUniqueId> connection_ids = pair.second;
+ total_connections += connection_ids.size();
+ Value hostname(pair.first.c_str(), document->GetAllocator());
+ client_host_json.AddMember("hostname", hostname, document->GetAllocator());
+ for (TUniqueId connection_id : connection_ids) {
+ ImpalaServer::ConnectionToSessionMap::iterator it =
+ server_->connection_to_sessions_map_.find(connection_id);
+ if (it != server_->connection_to_sessions_map_.end()) {
+ std::set<TUniqueId> session_ids = it->second;
+ for (TUniqueId session_id : session_ids) {
+ ImpalaServer::SessionStateMap::iterator session_state_map_iterator =
+ server_->session_state_map_.find(session_id);
+ if (session_state_map_iterator != server_->session_state_map_.end()
+ && client_hostname_sessions.find(session_id)
+ == client_hostname_sessions.end()) {
+ client_hostname_sessions.insert(session_id);
+ ++total_sessions;
+ shared_ptr<ImpalaServer::SessionState> session_state =
+ session_state_map_iterator->second;
+ if (session_state->expired || session_state->closed) {
+ ++total_inactive_sessions;
+ } else {
+ ++total_active_sessions;
+ }
+ inflight_queries += session_state->inflight_queries.size();
+ total_queries += session_state->total_queries;
+ }
+ }
+ }
+ }
+ client_host_json.AddMember("total_connections", total_connections,
+ document->GetAllocator());
+ client_host_json.AddMember("total_sessions", total_sessions,
+ document->GetAllocator());
+ client_host_json.AddMember("total_active_sessions", total_active_sessions,
+ document->GetAllocator());
+ client_host_json.AddMember("total_inactive_sessions", total_inactive_sessions,
+ document->GetAllocator());
+ client_host_json.AddMember("inflight_queries", inflight_queries,
+ document->GetAllocator());
+ client_host_json.AddMember("total_queries", total_queries, document->GetAllocator());
+ client_hosts.PushBack(client_host_json, document->GetAllocator());
+ }
+ document->AddMember("client_hosts", client_hosts, document->GetAllocator());
+}
+
+void ImpalaHttpHandler::FillConnectionsInfo(
+ Document* document, const ThriftServer::ConnectionContextList& connection_contexts) {
+ lock_guard<mutex> session_state_map_l(server_->session_state_map_lock_);
+ lock_guard<mutex> connection_to_sessions_map_l(
+ server_->connection_to_sessions_map_lock_);
+ Value connections(kArrayType);
+ int64_t num_beeswax_frontend_connections = 0;
+ int64_t num_hiveserver2_frontend_connections = 0;
+ int64_t num_hiveserver2_http_frontend_connections = 0;
+ int64_t num_external_frontend_connections = 0;
+ for (const ThriftServer::ConnectionContextList::value_type& connection_context :
+ connection_contexts) {
+ if (connection_context.get()) {
+ if (connection_context->server_name == ImpalaServer::BEESWAX_SERVER_NAME) {
+ ++num_beeswax_frontend_connections;
+ } else if (connection_context->server_name == ImpalaServer::HS2_SERVER_NAME) {
+ ++num_hiveserver2_frontend_connections;
+ } else if (connection_context->server_name == ImpalaServer::HS2_HTTP_SERVER_NAME) {
+ ++num_hiveserver2_http_frontend_connections;
+ } else if (connection_context->server_name
+ == ImpalaServer::EXTERNAL_FRONTEND_SERVER_NAME) {
+ ++num_external_frontend_connections;
+ }
+ Value connection_json(kObjectType);
+ Value connection_id(PrintId(connection_context->connection_id).c_str(),
+ document->GetAllocator());
+ connection_json.AddMember("connection_id", connection_id, document->GetAllocator());
+
+ Value user(connection_context->username.c_str(), document->GetAllocator());
+ connection_json.AddMember("user", user, document->GetAllocator());
+
+ Value delegated_user(
+ connection_context->do_as_user.c_str(), document->GetAllocator());
+ connection_json.AddMember(
+ "delegated_user", delegated_user, document->GetAllocator());
+
+ Value network_address(
+ TNetworkAddressToString(connection_context->network_address).c_str(),
+ document->GetAllocator());
+ connection_json.AddMember(
+ "network_address", network_address, document->GetAllocator());
+
+ Value server_name(
+ connection_context->server_name.c_str(), document->GetAllocator());
+ connection_json.AddMember("server_name", server_name, document->GetAllocator());
+
+ std::set<TUniqueId> valid_session_ids;
+ ImpalaServer::ConnectionToSessionMap::iterator it =
+ server_->connection_to_sessions_map_.find(connection_context->connection_id);
+ if (it != server_->connection_to_sessions_map_.end()) {
+ // Filter out invalid session
+ for (TUniqueId session_id : it->second) {
+ if (server_->session_state_map_.find(session_id)
+ != server_->session_state_map_.end())
+ valid_session_ids.insert(session_id);
+ }
+ }
+ Value session_ids_str(PrintIdSet(valid_session_ids, "\n").c_str(),
+ document->GetAllocator());
+ connection_json.AddMember("session_ids", session_ids_str, document->GetAllocator());
+ connections.PushBack(connection_json, document->GetAllocator());
+ }
+ }
+ document->AddMember("connections", connections, document->GetAllocator());
+ document->AddMember(
+ "num_connections", connection_contexts.size(), document->GetAllocator());
+ document->AddMember("num_beeswax_frontend_connections",
+ num_beeswax_frontend_connections, document->GetAllocator());
+ document->AddMember("num_hiveserver2_frontend_connections",
+ num_hiveserver2_frontend_connections, document->GetAllocator());
+ document->AddMember("num_hiveserver2_http_frontend_connections",
+ num_hiveserver2_http_frontend_connections, document->GetAllocator());
+ document->AddMember("num_external_frontend_connections",
+ num_external_frontend_connections, document->GetAllocator());
+}
+
void ImpalaHttpHandler::CatalogHandler(const Webserver::WebRequest& req,
Document* document) {
TGetDbsResult get_dbs_result;
diff --git a/be/src/service/impala-http-handler.h b/be/src/service/impala-http-handler.h
index 3e4a3f03b..ebbdd9570 100644
--- a/be/src/service/impala-http-handler.h
+++ b/be/src/service/impala-http-handler.h
@@ -281,5 +281,16 @@ class ImpalaHttpHandler {
/// supplied argument. Produces no JSON output.
void ResetResourcePoolStatsHandler(
const Webserver::WebRequest& req, rapidjson::Document* document);
+
+ /// Fill the sessions information into the document.
+ void FillSessionsInfo(rapidjson::Document* document);
+
+ /// Fill the client hosts information into the document.
+ void FillClientHostsInfo(rapidjson::Document* document,
+ const ThriftServer::ConnectionContextList& connection_contexts);
+
+ /// Fill the connections information into the document.
+ void FillConnectionsInfo(rapidjson::Document* document,
+ const ThriftServer::ConnectionContextList& connection_contexts);
};
}
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 58fa78ab4..d054a5413 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -453,9 +453,10 @@ const string LINEAGE_LOG_FILE_PREFIX = "impala_lineage_log_1.0-";
const uint32_t MAX_CANCELLATION_QUEUE_SIZE = 65536;
-const string BEESWAX_SERVER_NAME = "beeswax-frontend";
-const string HS2_SERVER_NAME = "hiveserver2-frontend";
-const string HS2_HTTP_SERVER_NAME = "hiveserver2-http-frontend";
+const string ImpalaServer::BEESWAX_SERVER_NAME = "beeswax-frontend";
+const string ImpalaServer::HS2_SERVER_NAME = "hiveserver2-frontend";
+const string ImpalaServer::HS2_HTTP_SERVER_NAME = "hiveserver2-http-frontend";
+const string ImpalaServer::EXTERNAL_FRONTEND_SERVER_NAME = "external-frontend";
const string ImpalaServer::DEFAULT_EXECUTOR_GROUP_NAME = "default";
@@ -3428,4 +3429,25 @@ __attribute__((noinline)) int ImpalaServer::SecretArg::ConstantTimeCompare(
// version has it.
return (secret_.hi != other.hi) + (secret_.lo != other.lo);
}
+
+void ImpalaServer::GetAllConnectionContexts(
+ ThriftServer::ConnectionContextList* connection_contexts) {
+ DCHECK_EQ(connection_contexts->size(), 0);
+ // Get the connection contexts of the beeswax server
+ if (beeswax_server_.get()) {
+ beeswax_server_->GetConnectionContextList(connection_contexts);
+ }
+ // Get the connection contexts of the hs2 server
+ if (hs2_server_.get()) {
+ hs2_server_->GetConnectionContextList(connection_contexts);
+ }
+ // Get the connection contexts of the hs2-http server
+ if (hs2_http_server_.get()) {
+ hs2_http_server_->GetConnectionContextList(connection_contexts);
+ }
+ // Get the connection contexts of the external fe server
+ if (external_fe_server_.get()) {
+ external_fe_server_->GetConnectionContextList(connection_contexts);
+ }
+}
}
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 19941f80a..cc3ff88c6 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -521,6 +521,10 @@ class ImpalaServer : public ImpalaServiceIf,
/// Returns true if 'user' was configured as an authorized proxy user.
bool IsAuthorizedProxyUser(const std::string& user) WARN_UNUSED_RESULT;
+ /// Gets connection contexts for all types of thrift servers.
+ void GetAllConnectionContexts(
+ ThriftServer::ConnectionContextList* connection_contexts);
+
// Mapping between query option names and levels
QueryOptionLevels query_option_levels_;
@@ -659,8 +663,11 @@ class ImpalaServer : public ImpalaServiceIf,
friend class ImpalaServerTest;
friend class QueryDriver;
+ static const string BEESWAX_SERVER_NAME;
+ static const string HS2_SERVER_NAME;
+ static const string HS2_HTTP_SERVER_NAME;
// Used to identify external frontend RPC calls
- const string EXTERNAL_FRONTEND_SERVER_NAME = "external-frontend";
+ static const string EXTERNAL_FRONTEND_SERVER_NAME;
boost::scoped_ptr<ImpalaHttpHandler> http_handler_;
diff --git a/be/src/util/debug-util.cc b/be/src/util/debug-util.cc
index be4a87d1d..927319de8 100644
--- a/be/src/util/debug-util.cc
+++ b/be/src/util/debug-util.cc
@@ -97,6 +97,18 @@ void PrintIdCompromised(const TUniqueId& id, char out[TUniqueIdBufferSize],
my_i64tohex(id.lo, out+17);
}
+string PrintIdSet(const std::set<TUniqueId>& ids, std::string separator) {
+ stringstream out;
+ auto it = ids.begin();
+ while (it != ids.end()) {
+ out << PrintId(*it);
+ if (++it != ids.end()) {
+ out << separator;
+ }
+ }
+ return out.str();
+}
+
bool ParseId(const string& s, TUniqueId* id) {
// For backwards compatibility, this method parses two forms of query ID from text:
// - <hex-int64_t><colon><hex-int64_t> - this format is the standard going forward
diff --git a/be/src/util/debug-util.h b/be/src/util/debug-util.h
index 66fa8e7ff..4179eefcd 100644
--- a/be/src/util/debug-util.h
+++ b/be/src/util/debug-util.h
@@ -65,6 +65,7 @@ std::string PrintBatch(RowBatch* batch);
/// printf "%lx:%lx\n", id.hi, id.lo
std::string PrintId(const TUniqueId& id, const std::string& separator = ":");
std::string PrintId(const UniqueIdPB& id, const std::string& separator = ":");
+std::string PrintIdSet(const std::set<TUniqueId>& ids, std::string separator = ",");
/// Converts id to a string representation without using any shared library calls.
/// Follows Breakpad's guidance for compromised contexts, see
diff --git a/www/sessions.tmpl b/www/sessions.tmpl
index 57f121541..6216c25c5 100644
--- a/www/sessions.tmpl
+++ b/www/sessions.tmpl
@@ -17,6 +17,68 @@ specific language governing permissions and limitations
under the License.
-->
{{> www/common-header.tmpl }}
+<h2>Client Hosts</h2>
+<table id="client-hosts-tbl" class='table table-bordered table-hover table-condensed'>
+ <thead>
+ <tr>
+ <th><small>Hostname</small></th>
+ <th><small>Total Connections</small></th>
+ <th><small>Total Sessions</small></th>
+ <th><small>Total Active Sessions</small></th>
+ <th><small>Total Inactive Sessions</small></th>
+ <th><small>Open Queries</small></th>
+ <th><small>Total Queries</small></th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#client_hosts}}
+ <tr>
+ <td><small>{{hostname}}</small></td>
+ <td><small>{{total_connections}}</small></td>
+ <td><small>{{total_sessions}}</small></td>
+ <td><small>{{total_active_sessions}}</small></td>
+ <td><small>{{total_inactive_sessions}}</small></td>
+ <td><small>{{inflight_queries}}</small></td>
+ <td><small>{{total_queries}}</small></td>
+ </tr>
+ {{/client_hosts}}
+ </tbody>
+</table>
+
+<h2>Connections</h2>
+<div class="alert alert-info" role="alert">
+<h4>There are {{num_connections}} connections, of which
+{{num_beeswax_frontend_connections}} are beeswax-frontend connections,
+{{num_hiveserver2_frontend_connections}} are hiveserver2-frontend connections,
+{{num_hiveserver2_http_frontend_connections}} are
+hiveserver2-http-frontend connections and {{num_external_frontend_connections}}
+are external-frontend connections
+</h4>
+</div>
+<table id="connections-tbl" class='table table-bordered table-hover table-condensed'>
+ <thead>
+ <tr>
+ <th><small>Connection ID</small></th>
+ <th><small>Connection Type</small></th>
+ <th><small>User</small></th>
+ <th><small>Delegated User</small></th>
+ <th><small>Network Address</small></th>
+ <th><small>Session IDs</small></th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#connections}}
+ <tr>
+ <td><small>{{connection_id}}</small></td>
+ <td><small>{{server_name}}</small></td>
+ <td><small>{{user}}</small></td>
+ <td><small>{{delegated_user}}</small></td>
+ <td><small>{{network_address}}</small></td>
+ <td style="white-space: pre-wrap;"><small>{{session_ids}}</small></td>
+ </tr>
+ {{/connections}}
+ </tbody>
+</table>
<h2>Sessions</h2>
@@ -40,7 +102,7 @@ which the session is removed from this list.</div>
<th><small>User</small></th>
<th><small>Delegated User</small></th>
<th><small>Session ID</small></th>
- <th><small>Network Address</small></th>
+ <th><small>Connection IDs</small></th>
<th><small>Default Database</small></th>
<th><small>Start Time</small></th>
<th><small>Last Accessed</small></th>
@@ -60,7 +122,7 @@ which the session is removed from this list.</div>
<td><small>{{user}}</small></td>
<td><small>{{delegated_user}}</small></td>
<td><small>{{session_id}}</small></td>
- <td><small>{{network_address}}</small></td>
+ <td style="white-space: pre-wrap;"><small>{{connection_ids}}</small></td>
<td><small>{{default_database}}</small></td>
<td data-order="{{start_time_sort}}"><small>{{start_time}}</small></td>
<td data-order="{{last_accessed_sort}}"><small>{{last_accessed}}</small></td>
@@ -74,6 +136,24 @@ which the session is removed from this list.</div>
</tbody>
</table>
+<script>
+ $(document).ready(function() {
+ $('#client-hosts-tbl').DataTable({
+ "order": [[ 1, "desc" ]],
+ "pageLength": 100
+ });
+ });
+</script>
+
+<script>
+ $(document).ready(function() {
+ $('#connections-tbl').DataTable({
+ "order": [[ 4, "desc" ]],
+ "pageLength": 100
+ });
+ });
+</script>
+
<script>
$(document).ready(function() {
$('#sessions-tbl').DataTable({