You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by jd...@apache.org on 2021/02/12 12:17:16 UTC
[qpid-dispatch] branch master updated: DISPATCH-1586 - Add
allocator metrics (#696)
This is an automated email from the ASF dual-hosted git repository.
jdanek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/qpid-dispatch.git
The following commit(s) were added to refs/heads/master by this push:
new 4a30f53 DISPATCH-1586 - Add allocator metrics (#696)
4a30f53 is described below
commit 4a30f537971d929c3ba63019acefd2997c4eb762
Author: Fernando Giorgetti <fg...@users.noreply.github.com>
AuthorDate: Fri Feb 12 09:17:06 2021 -0300
DISPATCH-1586 - Add allocator metrics (#696)
---
src/http-libwebsockets.c | 140 +++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 136 insertions(+), 4 deletions(-)
diff --git a/src/http-libwebsockets.c b/src/http-libwebsockets.c
index 9a2e803..66cf187 100644
--- a/src/http-libwebsockets.c
+++ b/src/http-libwebsockets.c
@@ -460,6 +460,11 @@ typedef struct metric_definition {
int_metric value;
} metric_definition;
+typedef struct allocator_metric_definition {
+ const char* name;
+ qd_alloc_stats_t *(*fn)(void);
+} allocator_metric_definition;
+
static int stats_get_connections(qdr_global_stats_t *stats) { return stats->connections; }
static int stats_get_links(qdr_global_stats_t *stats) { return stats->links; }
static int stats_get_addrs(qdr_global_stats_t *stats) { return stats->addrs; }
@@ -483,6 +488,43 @@ static int stats_get_deliveries_stuck(qdr_global_stats_t *stats) { return stats-
static int stats_get_links_blocked(qdr_global_stats_t *stats) { return stats->links_blocked; }
static int stats_get_deliveries_redirected_to_fallback(qdr_global_stats_t *stats) { return stats->deliveries_redirected_to_fallback; }
+qd_alloc_stats_t *alloc_stats_qd_bitmask_t(void);
+qd_alloc_stats_t *alloc_stats_qd_buffer_t(void);
+qd_alloc_stats_t *alloc_stats_qd_composed_field_t(void);
+qd_alloc_stats_t *alloc_stats_qd_composite_t(void);
+qd_alloc_stats_t *alloc_stats_qd_connection_t(void);
+qd_alloc_stats_t *alloc_stats_qd_hash_handle_t(void);
+qd_alloc_stats_t *alloc_stats_qd_hash_item_t(void);
+qd_alloc_stats_t *alloc_stats_qd_iterator_t(void);
+qd_alloc_stats_t *alloc_stats_qd_link_ref_t(void);
+qd_alloc_stats_t *alloc_stats_qd_link_t(void);
+qd_alloc_stats_t *alloc_stats_qd_listener_t(void);
+qd_alloc_stats_t *alloc_stats_qd_log_entry_t(void);
+qd_alloc_stats_t *alloc_stats_qd_management_context_t(void);
+qd_alloc_stats_t *alloc_stats_qd_message_content_t(void);
+qd_alloc_stats_t *alloc_stats_qd_message_t(void);
+qd_alloc_stats_t *alloc_stats_qd_node_t(void);
+qd_alloc_stats_t *alloc_stats_qd_parse_node_t(void);
+qd_alloc_stats_t *alloc_stats_qd_parsed_field_t(void);
+qd_alloc_stats_t *alloc_stats_qd_timer_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_action_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_address_config_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_address_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_connection_info_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_connection_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_connection_work_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_core_timer_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_delivery_cleanup_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_delivery_ref_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_delivery_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_field_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_general_work_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_link_ref_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_link_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_link_work_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_query_t(void);
+qd_alloc_stats_t *alloc_stats_qdr_terminus_t(void);
+
static struct metric_definition metrics[] = {
{"qdr_connections_total", "gauge", stats_get_connections},
{"qdr_links_total", "gauge", stats_get_links},
@@ -509,9 +551,60 @@ static struct metric_definition metrics[] = {
};
static size_t metrics_length = sizeof(metrics)/sizeof(metrics[0]);
+static struct allocator_metric_definition allocator_metrics[] = {
+ {"qdr_allocator_qd_bitmask_t", alloc_stats_qd_bitmask_t},
+ {"qdr_allocator_qd_buffer_t", alloc_stats_qd_buffer_t},
+ {"qdr_allocator_qd_composed_field_t", alloc_stats_qd_composed_field_t},
+ {"qdr_allocator_qd_composite_t", alloc_stats_qd_composite_t},
+ {"qdr_allocator_qd_connection_t", alloc_stats_qd_connection_t},
+ {"qdr_allocator_qd_hash_handle_t", alloc_stats_qd_hash_handle_t},
+ {"qdr_allocator_qd_hash_item_t", alloc_stats_qd_hash_item_t},
+ {"qdr_allocator_qd_iterator_t", alloc_stats_qd_iterator_t},
+ {"qdr_allocator_qd_link_ref_t", alloc_stats_qd_link_ref_t},
+ {"qdr_allocator_qd_link_t", alloc_stats_qd_link_t},
+ {"qdr_allocator_qd_listener_t", alloc_stats_qd_listener_t},
+ {"qdr_allocator_qd_log_entry_t", alloc_stats_qd_log_entry_t},
+ {"qdr_allocator_qd_management_context_t", alloc_stats_qd_management_context_t},
+ {"qdr_allocator_qd_message_content_t", alloc_stats_qd_message_content_t},
+ {"qdr_allocator_qd_message_t", alloc_stats_qd_message_t},
+ {"qdr_allocator_qd_node_t", alloc_stats_qd_node_t},
+ {"qdr_allocator_qd_parse_node_t", alloc_stats_qd_parse_node_t},
+ {"qdr_allocator_qd_parsed_field_t", alloc_stats_qd_parsed_field_t},
+ {"qdr_allocator_qd_timer_t", alloc_stats_qd_timer_t},
+ {"qdr_allocator_qdr_action_t", alloc_stats_qdr_action_t},
+ {"qdr_allocator_qdr_address_config_t", alloc_stats_qdr_address_config_t},
+ {"qdr_allocator_qdr_address_t", alloc_stats_qdr_address_t},
+ {"qdr_allocator_qdr_connection_info_t", alloc_stats_qdr_connection_info_t},
+ {"qdr_allocator_qdr_connection_t", alloc_stats_qdr_connection_t},
+ {"qdr_allocator_qdr_connection_work_t", alloc_stats_qdr_connection_work_t},
+ {"qdr_allocator_qdr_core_timer_t", alloc_stats_qdr_core_timer_t},
+ {"qdr_allocator_qdr_delivery_cleanup_t", alloc_stats_qdr_delivery_cleanup_t},
+ {"qdr_allocator_qdr_delivery_ref_t", alloc_stats_qdr_delivery_ref_t},
+ {"qdr_allocator_qdr_delivery_t", alloc_stats_qdr_delivery_t},
+ {"qdr_allocator_qdr_field_t", alloc_stats_qdr_field_t},
+ {"qdr_allocator_qdr_general_work_t", alloc_stats_qdr_general_work_t},
+ {"qdr_allocator_qdr_link_ref_t", alloc_stats_qdr_link_ref_t},
+ {"qdr_allocator_qdr_link_t", alloc_stats_qdr_link_t},
+ {"qdr_allocator_qdr_link_work_t", alloc_stats_qdr_link_work_t},
+ {"qdr_allocator_qdr_query_t", alloc_stats_qdr_query_t},
+ {"qdr_allocator_qdr_terminus_t", alloc_stats_qdr_terminus_t}
+};
+static size_t allocator_metrics_length = sizeof(allocator_metrics)/sizeof(allocator_metrics[0]);
+
+#define ALLOC_DATA(S, F) ((allocator_field) {#F, (S!=NULL? S->F: 0)})
+
+typedef struct allocator_field {
+ const char* name;
+ uint64_t value;
+} allocator_field;
+
static bool write_stats(uint8_t **position, const uint8_t * const end, const char* name, const char* type, int value)
{
//11 chars + type + 2*name + 20 chars for int
+ // average metric name size is 30 bytes
+ // average metric type size is 8 bytes
+ // current number of metrics is 22
+ // total metric buffer size = 22 * (11 + 8 + 2*30 + 20) = 2178
size_t length = 11 + strlen(type) + strlen(name)*2 + 20;
if (end - *position >= length) {
*position += lws_snprintf((char*) *position, end - *position, "# TYPE %s %s\n", name, type);
@@ -522,11 +615,38 @@ static bool write_stats(uint8_t **position, const uint8_t * const end, const cha
}
}
+static bool write_allocator_stats(uint8_t **position, const uint8_t * const end, const char* name, allocator_field field)
+{
+ // 30 chars (static) + 2*name + 2*field.name + 20 for int
+ // average allocator metric name size is 54 bytes (name:field.name)
+ // current number of metrics is 180
+ // total allocator buffer size = 180 * (30 + 2*54 + 20) = 28440
+ size_t length = 30 + strlen(name)*2 + strlen(field.name)*2 + 20;
+ if (end - *position >= length) {
+ *position += lws_snprintf((char*) *position, end - *position, "# TYPE %s:%s_bytes gauge\n", name, field.name);
+ *position += lws_snprintf((char*) *position, end - *position, "%s:%s_bytes %"PRIu64"\n", name, field.name, field.value);
+ return true;
+ } else {
+ return false;
+ }
+}
+
static bool write_metric(uint8_t **position, const uint8_t * const end, metric_definition* definition, qdr_global_stats_t* stats)
{
return write_stats(position, end, definition->name, definition->type, definition->value(stats));
}
+static bool write_allocator_metric(uint8_t **position, const uint8_t * const end, allocator_metric_definition* definition)
+{
+ qd_alloc_stats_t *allocator_stats = definition->fn();
+ if (!write_allocator_stats(position, end, definition->name, ALLOC_DATA(allocator_stats, total_alloc_from_heap))) return false;
+ if (!write_allocator_stats(position, end, definition->name, ALLOC_DATA(allocator_stats, total_free_to_heap))) return false;
+ if (!write_allocator_stats(position, end, definition->name, ALLOC_DATA(allocator_stats, held_by_threads))) return false;
+ if (!write_allocator_stats(position, end, definition->name, ALLOC_DATA(allocator_stats, batches_rebalanced_to_threads))) return false;
+ if (!write_allocator_stats(position, end, definition->name, ALLOC_DATA(allocator_stats, batches_rebalanced_to_global))) return false;
+ return true;
+}
+
static int add_header_by_name(struct lws *wsi, const char* name, const char* value, uint8_t** position, uint8_t* end)
{
return lws_add_http_header_by_name(wsi, (unsigned char*) name, (unsigned char*) value, strlen(value), position, end);
@@ -537,7 +657,8 @@ static int callback_metrics(struct lws *wsi, enum lws_callback_reasons reason,
{
qd_http_server_t *hs = wsi_server(wsi);
stats_t *stats = (stats_t*) user;
- uint8_t buffer[LWS_PRE + 2048];
+ // rationale for buffer size is explained at write_stats and write_allocator_stats
+ uint8_t buffer[LWS_PRE + 30618];
uint8_t *start = &buffer[LWS_PRE], *position = start, *end = &buffer[sizeof(buffer) - LWS_PRE - 1];
switch (reason) {
@@ -569,15 +690,26 @@ static int callback_metrics(struct lws *wsi, enum lws_callback_reasons reason,
stats->current++;
qd_log(hs->log, QD_LOG_DEBUG, "wrote metric %lu of %lu", stats->current, metrics_length);
} else {
- qd_log(hs->log, QD_LOG_DEBUG, "insufficient space in buffer");
+ qd_log(hs->log, QD_LOG_WARNING, "insufficient space in buffer");
break;
}
}
- int n = stats->current < metrics_length ? LWS_WRITE_HTTP : LWS_WRITE_HTTP_FINAL;
+
+ int alloc_cur = 0;
+ while (alloc_cur < allocator_metrics_length) {
+ if (write_allocator_metric(&position, end, &allocator_metrics[alloc_cur])) {
+ qd_log(hs->log, QD_LOG_DEBUG, "wrote allocator metric %lu of %lu", alloc_cur, allocator_metrics_length);
+ alloc_cur++;
+ } else {
+ qd_log(hs->log, QD_LOG_WARNING, "insufficient space in buffer");
+ break;
+ }
+ }
+ int n = (stats->current < metrics_length) || (alloc_cur < allocator_metrics_length) ? LWS_WRITE_HTTP : LWS_WRITE_HTTP_FINAL;
//write buffer
size_t available = position - start;
- if (lws_write(wsi, (unsigned char*) start, available, n) != available)
+ if (lws_write(wsi, (unsigned char*) start, available, n) != available)
return 1;
if (n == LWS_WRITE_HTTP_FINAL) {
if (lws_http_transaction_completed(wsi)) return -1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org