You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2026/05/12 13:49:57 UTC
(kudu) branch master updated: KUDU-3690: Add filtering to /metrics_prometheus
This is an automated email from the ASF dual-hosted git repository.
martongreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 3e42c0e7d KUDU-3690: Add filtering to /metrics_prometheus
3e42c0e7d is described below
commit 3e42c0e7dc2199f80eabd0413e53de79e3940f1b
Author: Marton Greber <gr...@gmail.com>
AuthorDate: Mon May 4 17:01:25 2026 +0200
KUDU-3690: Add filtering to /metrics_prometheus
The /metrics_prometheus endpoint now accepts the same query
parameters as the /metrics JSON endpoint for filtering:
?level= severity threshold (debug/info/warn)
?types= entity type (server/tablet/table)
?ids= entity ID substring
?metrics= metric name substring
?attributes= key/value pairs for entity attribute matching
Malformed ?attributes= (odd number of values) returns HTTP 400,
mirroring the existing JSON endpoint behaviour.
Change-Id: I5c0b23ae5c184bf9e33e453736cef5e7ce8ee2e1
Reviewed-on: http://gerrit.cloudera.org:8080/24269
Reviewed-by: Zoltan Chovan <zc...@cloudera.com>
Reviewed-by: Ashwani Raina <ar...@gmail.com>
Tested-by: Marton Greber <gr...@gmail.com>
---
src/kudu/master/master-test.cc | 270 ++++++++++++++++++++++++++++
src/kudu/server/default_path_handlers.cc | 54 +++---
src/kudu/tserver/tablet_server-test.cc | 299 +++++++++++++++++++++++++++++++
src/kudu/util/metrics-test.cc | 193 ++++++++++++++++++++
src/kudu/util/metrics.h | 7 +-
src/kudu/util/prometheus_writer.cc | 4 +
src/kudu/util/test_util.cc | 8 +
src/kudu/util/test_util.h | 5 +
8 files changed, 816 insertions(+), 24 deletions(-)
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 3214914d3..934ab749f 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -22,6 +22,7 @@
#include <cstdint>
#include <ctime>
#include <functional>
+#include <initializer_list>
#include <map>
#include <memory>
#include <numeric>
@@ -4113,6 +4114,275 @@ TEST_F(MasterTest, PrometheusMetricsLevelFiltering) {
}
}
+// Verify that the ?level= query parameter on /metrics_prometheus overrides
+// the --metrics_default_level flag for a single request.
+TEST_F(MasterTest, PrometheusMetricsLevelQueryParam) {
+ constexpr char kTableName[] = "prom_level_query_param";
+ const Schema kTableSchema(
+ { ColumnSchema("key", INT32), ColumnSchema("v1", UINT64) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ // Set the flag to "debug" so all levels are emitted by default.
+ google::FlagSaver saver;
+ FLAGS_metrics_default_level = "debug";
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ {
+ // ?level=warn overrides the flag: only warn-level metrics should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?level=warn", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+ }
+ {
+ // ?level=info overrides the flag: debug metrics absent, info+warn present.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?level=info", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+ }
+}
+
+// Verify that ?types= filters the output to only entities of the given type.
+TEST_F(MasterTest, PrometheusMetricsTypeFiltering) {
+ constexpr char kTableName[] = "prom_type_filter";
+ const Schema kTableSchema(
+ { ColumnSchema("key", INT32) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ {
+ // ?types=server: only server-entity metrics should appear; tablet metrics absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?types=server", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running "); // server-level metric
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // tablet-level metric
+ }
+ {
+ // ?types=tablet: only tablet-entity metrics should appear; server metrics absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?types=tablet", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "raft_term "); // tablet-level metric
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // server-level metric
+ }
+ // TODO(KUDU-3774): add a ?types=table case.
+ {
+ // ?types=nonexistent_type: no metrics should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?types=nonexistent_type", &buf));
+ // No entity type matches, so the output should contain no metric value lines at all.
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+ }
+}
+
+// Verify that ?metrics= filters the output to only metrics whose names
+// contain the given substring.
+TEST_F(MasterTest, PrometheusMetricsNameFiltering) {
+ constexpr char kTableName[] = "prom_name_filter";
+ const Schema kTableSchema(
+ { ColumnSchema("key", INT32) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ {
+ // ?metrics=threads_running: only that metric should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?metrics=threads_running", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running ");
+ ASSERT_STR_NOT_MATCHES(str, "rpcs_queue_overflow ");
+ }
+ {
+ // ?metrics=nonexistent: no metrics should be emitted.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?metrics=nonexistent_metric_xyz", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+ }
+}
+
+// Verify that ?ids= filters the output to only entities with the given ID.
+TEST_F(MasterTest, PrometheusMetricsIdFiltering) {
+ constexpr char kTableName[] = "prom_id_filter";
+ const Schema kTableSchema(
+ { ColumnSchema("key", INT32) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ {
+ // ?ids=kudu.master: only server-entity metrics should appear (tablet metrics absent).
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?ids=kudu.master", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running "); // server metric
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // tablet metric
+ }
+ {
+ // ?ids=nonexistent_id: no metrics should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?ids=nonexistent_id", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+ }
+}
+
+// Verify that two different filters can be combined in a single request.
+TEST_F(MasterTest, PrometheusMetricsCombinedFilters) {
+ constexpr char kTableName[] = "prom_combined_filter";
+ const Schema kTableSchema(
+ { ColumnSchema("key", INT32), ColumnSchema("v1", UINT64) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ // ?level=info&types=server: should include info/warn server metrics
+ // but exclude debug-level metrics and any tablet-entity metrics.
+ ASSERT_OK(c.FetchURL(base_url + "?level=info&types=server", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // debug-level, tablet entity
+ ASSERT_STR_MATCHES(str, "threads_running "); // info-level, server entity
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, server entity
+}
+
+// Verify that the ?attributes= filter returns HTTP 400 when an odd number of
+// values is supplied (attribute keys and values must come in pairs).
+// This applies to both the JSON and Prometheus metrics endpoints.
+TEST_F(MasterTest, MetricsOddAttributesReturnsBadRequest) {
+ const string base_prom_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ const string base_json_url = Substitute("http://$0/metrics",
+ mini_master_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ // One value instead of the required even number of values.
+ Status s = c.FetchURL(base_prom_url + "?attributes=table_id", &buf);
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "HTTP 400");
+
+ s = c.FetchURL(base_json_url + "?attributes=table_id", &buf);
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "HTTP 400");
+}
+
+// Verify that query parameters that are meaningful only for the JSON metrics
+// endpoint (?include_raw_histograms, ?include_schema, ?compact) are silently
+// ignored when passed to /metrics_prometheus, and do not cause a crash or
+// suppress any metrics. Unrecognized filter keys are also silently ignored.
+TEST_F(MasterTest, PrometheusMetricsJsonOnlyParamsIgnored) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(
+ base_url + "?include_raw_histograms=1&include_schema=1&compact=1&unknown_key=foo",
+ &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running "); // info-level, server entity
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, server entity
+}
+
+// Verify that supplying an empty value for a recognized filter key is treated
+// as a no-op: the empty string matches every value via substring logic, so all
+// metrics are included and nothing crashes.
+// Similarly, ?level= with an empty value falls back to the default level,
+// again returning all metrics.
+TEST_F(MasterTest, PrometheusMetricsEmptyFilterValuesAreNoOp) {
+ constexpr char kTableName[] = "prom_empty_filter";
+ const Schema kTableSchema({ ColumnSchema("key", INT32) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+
+ // Each of these supplies an empty value for a recognized filter key.
+ // None of them should crash, and all should still emit well-known metrics.
+ for (const char* query : {"?types=", "?ids=", "?metrics=", "?level="}) {
+ ASSERT_OK(c.FetchURL(base_url + query, &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running ") << "query: " << query;
+ }
+}
+
+// Verify that a valid filter combined with invalid input (unrecognized key,
+// garbage level value, or empty value for another key) still applies the valid
+// filter correctly and does not crash.
+TEST_F(MasterTest, PrometheusMetricsValidFilterWithBadInputIgnored) {
+ constexpr char kTableName[] = "prom_valid_bad_filter";
+ const Schema kTableSchema({ ColumnSchema("key", INT32) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+
+ {
+ // Valid level filter + unrecognized key: level filter must still apply.
+ ASSERT_OK(c.FetchURL(base_url + "?level=info&unknown_key=foo", &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // debug-level, absent at info
+ ASSERT_STR_MATCHES(str, "threads_running "); // info-level, present
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, present
+ }
+ {
+ // Valid types filter + garbage level value: types filter must still apply,
+ // garbage level falls back to kDebug so all levels are included.
+ ASSERT_OK(c.FetchURL(base_url + "?types=server&level=garbage", &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // tablet entity, absent
+ ASSERT_STR_MATCHES(str, "threads_running "); // server entity, present
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level server entity, also present
+ }
+ {
+ // Valid level filter + empty types value: level filter must still apply,
+ // empty types matches all entity types so nothing is filtered by type.
+ ASSERT_OK(c.FetchURL(base_url + "?level=warn&types=", &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // debug-level, absent at warn
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // info-level, absent at warn
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, present
+ }
+}
+
// Test that the Prometheus service discovery endpoint returns proper target groups
// for masters and tservers with all expected fields (group, scheme, cluster_id, location).
TEST_F(MasterTest, TestPrometheusServiceDiscoveryEndpoint) {
diff --git a/src/kudu/server/default_path_handlers.cc b/src/kudu/server/default_path_handlers.cc
index 1e686adee..5ff258dcb 100644
--- a/src/kudu/server/default_path_handlers.cc
+++ b/src/kudu/server/default_path_handlers.cc
@@ -92,8 +92,8 @@ DEFINE_string(metrics_default_level, "debug",
"both in JSON and Prometheus formats. Valid choices are 'debug', "
"'info', and 'warn'. The levels are ordered and lower levels "
"include the levels above them. This value can be overridden "
- "by passing the level query parameter to the '/metrics' endpoint "
- "when fetching metrics in JSON format.");
+ "per-request by passing the 'level' query parameter to the "
+ "'/metrics' or '/metrics_prometheus' endpoints.");
TAG_FLAG(metrics_default_level, advanced);
TAG_FLAG(metrics_default_level, runtime);
TAG_FLAG(metrics_default_level, evolving);
@@ -486,6 +486,24 @@ static vector<string> ParseArray(const Webserver::ArgumentMap& args, const strin
return value;
}
+static bool ParseMetricFilters(const Webserver::WebRequest& req,
+ Webserver::PrerenderedWebResponse* resp,
+ MetricFilters* filters) {
+ filters->entity_types = ParseArray(req.parsed_args, "types");
+ filters->entity_ids = ParseArray(req.parsed_args, "ids");
+ filters->entity_attrs = ParseArray(req.parsed_args, "attributes");
+ filters->entity_metrics = ParseArray(req.parsed_args, "metrics");
+ filters->entity_level = FindWithDefault(req.parsed_args, "level",
+ FLAGS_metrics_default_level);
+ if (filters->entity_attrs.size() % 2 != 0) {
+ resp->status_code = HttpStatusCode::BadRequest;
+ resp->output << "The 'attributes' parameter must be provided as key-value pairs "
+ "(i.e. an even number of values)";
+ return false;
+ }
+ return true;
+}
+
static void WriteMetricsAsJson(const MetricRegistry* const metrics,
const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
@@ -493,12 +511,12 @@ static void WriteMetricsAsJson(const MetricRegistry* const metrics,
opts.include_raw_histograms = ParseBool(req.parsed_args, "include_raw_histograms");
opts.include_schema_info = ParseBool(req.parsed_args, "include_schema");
- MetricFilters& filters = opts.filters;
- filters.entity_types = ParseArray(req.parsed_args, "types");
- filters.entity_ids = ParseArray(req.parsed_args, "ids");
- filters.entity_attrs = ParseArray(req.parsed_args, "attributes");
- filters.entity_metrics = ParseArray(req.parsed_args, "metrics");
- filters.entity_level = FindWithDefault(req.parsed_args, "level", FLAGS_metrics_default_level);
+ // ParseMetricFilters() sets resp->status_code to HTTP 400 and returns false
+ // on malformed input (e.g. an odd number of ?attributes= values).
+ if (!ParseMetricFilters(req, resp, &opts.filters)) {
+ return;
+ }
+
vector<string> merge_rules = ParseArray(req.parsed_args, "merge_rules");
for (const auto& merge_rule : merge_rules) {
vector<string> values;
@@ -513,24 +531,20 @@ static void WriteMetricsAsJson(const MetricRegistry* const metrics,
JsonWriter::Mode json_mode = ParseBool(req.parsed_args, "compact") ?
JsonWriter::COMPACT : JsonWriter::PRETTY;
-
- // The number of entity_attrs should always be even because
- // each pair represents a key and a value.
- if (filters.entity_attrs.size() % 2 != 0) {
- resp->status_code = HttpStatusCode::BadRequest;
- WARN_NOT_OK(Status::InvalidArgument(""), "The parameter of 'attributes' is wrong");
- } else {
- JsonWriter writer(&resp->output, json_mode);
- WARN_NOT_OK(metrics->WriteAsJson(&writer, opts), "Couldn't write JSON metrics over HTTP");
- }
+ JsonWriter writer(&resp->output, json_mode);
+ WARN_NOT_OK(metrics->WriteAsJson(&writer, opts), "Couldn't write JSON metrics over HTTP");
}
static void WriteMetricsAsPrometheus(const MetricRegistry* const metrics,
const string& hostname,
- const Webserver::WebRequest& /*req*/,
+ const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
MetricPrometheusOptions opts;
- opts.filters.entity_level = FLAGS_metrics_default_level;
+ // ParseMetricFilters() sets resp->status_code to HTTP 400 and returns false
+ // on malformed input (e.g. an odd number of ?attributes= values).
+ if (!ParseMetricFilters(req, resp, &opts.filters)) {
+ return;
+ }
if (FLAGS_metrics_prometheus_use_entity_labels) {
opts.hostname = hostname;
}
diff --git a/src/kudu/tserver/tablet_server-test.cc b/src/kudu/tserver/tablet_server-test.cc
index 4ea10a638..a7c5e9d53 100644
--- a/src/kudu/tserver/tablet_server-test.cc
+++ b/src/kudu/tserver/tablet_server-test.cc
@@ -4465,6 +4465,305 @@ TEST_F(TabletServerTest, PrometheusMetricsLevelFiltering) {
}
}
+// Verify that the ?level= query parameter on /metrics_prometheus overrides
+// the --metrics_default_level flag for a single request.
+TEST_F(TabletServerTest, PrometheusMetricsLevelQueryParam) {
+ // Set the flag to "debug" so all levels are emitted by default.
+ google::FlagSaver saver;
+ FLAGS_metrics_default_level = "debug";
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ {
+ // ?level=warn overrides the flag: only warn-level metrics should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?level=warn", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+ }
+ {
+ // ?level=info overrides the flag: debug absent, info+warn present.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?level=info", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+ }
+}
+
+// Verify that ?types= filters the Prometheus output to only entities of the
+// given type.
+TEST_F(TabletServerTest, PrometheusMetricsTypeFiltering) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ {
+ // ?types=server: only server-entity metrics should appear; tablet metrics absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?types=server", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running "); // server-level metric
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // tablet-level metric
+ }
+ {
+ // ?types=tablet: only tablet-entity metrics should appear; server metrics absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?types=tablet", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "raft_term "); // tablet-level metric
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // server-level metric
+ }
+ // TODO(KUDU-3774): add a ?types=table case.
+ {
+ // ?types=nonexistent_type: no metrics should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?types=nonexistent_type", &buf));
+ // No entity type matches, so the output should contain no metric value lines at all.
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+ }
+}
+
+// Verify that ?metrics= filters the output to only metrics whose names
+// contain the given substring.
+TEST_F(TabletServerTest, PrometheusMetricsNameFiltering) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ {
+ // ?metrics=raft_term: only that metric should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?metrics=raft_term", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "raft_term ");
+ ASSERT_STR_NOT_MATCHES(str, "threads_running ");
+ }
+ {
+ // ?metrics=nonexistent: no metrics should be emitted.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?metrics=nonexistent_metric_xyz", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+ }
+}
+
+// Verify that ?ids= filters the output to only entities with the given ID.
+TEST_F(TabletServerTest, PrometheusMetricsIdFiltering) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ {
+ // ?ids=<tablet-id>: only tablet-entity metrics should appear; server metrics absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + Substitute("?ids=$0", kTabletId), &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "raft_term "); // tablet metric
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // server metric
+ }
+ {
+ // ?ids=nonexistent_id: no metrics should appear.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?ids=nonexistent_id_xyz", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+ }
+}
+
+// Verify that ?attributes= filters the Prometheus output to only entities
+// whose attributes match the given key/value pair.
+// This test creates two tablets belonging to different tables and verifies
+// that filtering by table_id returns only metrics for the matching table's tablets.
+TEST_F(TabletServerTest, PrometheusMetricsAttributeFiltering) {
+ // The default tablet (kTabletId) belongs to kTableId == "TestTable".
+ // Add a second tablet belonging to a different table.
+ constexpr char kOtherTableId[] = "OtherTable";
+ constexpr char kOtherTabletId[] = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee";
+ ASSERT_OK(mini_server_->AddTestTablet(kOtherTableId, kOtherTabletId, schema_));
+ ASSERT_OK(WaitForTabletRunning(kOtherTabletId));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ {
+ // Filter by table_id=TestTable: the default tablet should appear,
+ // but the OtherTable tablet must be absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?attributes=table_id,TestTable", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_CONTAINS(str, kTabletId);
+ ASSERT_STR_NOT_CONTAINS(str, kOtherTabletId);
+ }
+ {
+ // Filter by table_id=OtherTable: the second tablet should appear,
+ // but the default tablet must be absent.
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(base_url + "?attributes=table_id,OtherTable", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_CONTAINS(str, kOtherTabletId);
+ ASSERT_STR_NOT_CONTAINS(str, kTabletId);
+ }
+}
+
+// Verify that combining ?attributes= with ?ids= where the ID belongs to a
+// tablet from a *different* table returns an empty Prometheus output.
+// This guards against a scenario where the two filters interact unexpectedly.
+TEST_F(TabletServerTest, PrometheusMetricsMixedAttributesAndIds) {
+ constexpr char kOtherTableId[] = "OtherTable";
+ constexpr char kOtherTabletId[] = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee";
+ ASSERT_OK(mini_server_->AddTestTablet(kOtherTableId, kOtherTabletId, schema_));
+ ASSERT_OK(WaitForTabletRunning(kOtherTabletId));
+
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ // Ask for table_id=TestTable but the ID of the OtherTable tablet.
+ // The attribute filter and the id filter are ANDed: no entity matches both,
+ // so the output should be empty of metric value lines.
+ ASSERT_OK(c.FetchURL(
+ base_url + Substitute("?attributes=table_id,TestTable&ids=$0", kOtherTabletId),
+ &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ NO_FATALS(CheckNoPrometheusValueLines(str));
+}
+
+// Verify that two different filters can be combined in a single request.
+// For example, ?level=info&types=tablet should output tablet-entity metrics
+// at info level or above, with no server-entity metrics and no debug-level metrics.
+TEST_F(TabletServerTest, PrometheusMetricsCombinedFilters) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ // ?level=info&types=tablet: only tablet entities, only info-and-above metrics.
+ ASSERT_OK(c.FetchURL(base_url + "?level=info&types=tablet", &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "on_disk_data_size "); // info-level, tablet entity
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // debug-level, tablet entity
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // info-level, but server entity
+}
+
+// Verify that the ?attributes= filter returns HTTP 400 when an odd number of
+// values is supplied (attribute keys and values must come in pairs).
+// This applies to both the JSON and Prometheus metrics endpoints.
+TEST_F(TabletServerTest, MetricsOddAttributesReturnsBadRequest) {
+ const string base_prom_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ const string base_json_url = Substitute("http://$0/metrics",
+ mini_server_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ Status s = c.FetchURL(base_prom_url + "?attributes=table_id", &buf);
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "HTTP 400");
+
+ s = c.FetchURL(base_json_url + "?attributes=table_id", &buf);
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "HTTP 400");
+}
+
+// Verify that query parameters that are meaningful only for the JSON metrics
+// endpoint (?include_raw_histograms, ?include_schema, ?compact) are silently
+// ignored when passed to /metrics_prometheus, and do not cause a crash or
+// suppress any metrics. Unrecognized filter keys are also silently ignored.
+TEST_F(TabletServerTest, PrometheusMetricsJsonOnlyParamsIgnored) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(
+ base_url + "?include_raw_histograms=1&include_schema=1&compact=1&unknown_key=foo",
+ &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running "); // info-level, server entity
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, server entity
+}
+
+// Verify that supplying an empty value for a recognized filter key is treated
+// as a no-op: the empty string matches every value via substring logic, so all
+// metrics are included and nothing crashes.
+// Similarly, ?level= with an empty value falls back to the default level,
+// again returning all metrics.
+TEST_F(TabletServerTest, PrometheusMetricsEmptyFilterValuesAreNoOp) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+
+ // Each of these supplies an empty value for a recognized filter key.
+ // None of them should crash, and all should still emit well-known metrics.
+ for (const char* query : {"?types=", "?ids=", "?metrics=", "?level="}) {
+ ASSERT_OK(c.FetchURL(base_url + query, &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_MATCHES(str, "threads_running ") << "query: " << query;
+ }
+}
+
+// Verify that a valid filter combined with invalid input (unrecognized key,
+// garbage level value, or empty value for another key) still applies the valid
+// filter correctly and does not crash.
+TEST_F(TabletServerTest, PrometheusMetricsValidFilterWithBadInputIgnored) {
+ const string base_url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ EasyCurl c;
+ faststring buf;
+
+ {
+ // Valid level filter + unrecognized key: level filter must still apply.
+ ASSERT_OK(c.FetchURL(base_url + "?level=info&unknown_key=foo", &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // debug-level, absent at info
+ ASSERT_STR_MATCHES(str, "threads_running "); // info-level, present
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, present
+ }
+ {
+ // Valid types filter + garbage level value: types filter must still apply,
+ // garbage level falls back to kDebug so all levels are included.
+ ASSERT_OK(c.FetchURL(base_url + "?types=server&level=garbage", &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // tablet entity, absent
+ ASSERT_STR_MATCHES(str, "threads_running "); // server entity, present
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level server entity, also present
+ }
+ {
+ // Valid level filter + empty types value: level filter must still apply,
+ // empty types matches all entity types so nothing is filtered by type.
+ ASSERT_OK(c.FetchURL(base_url + "?level=warn&types=", &buf));
+ const string& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // debug-level, absent at warn
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // info-level, absent at warn
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // warn-level, present
+ }
+}
+
// Test that hostname is set properly for TabletServer's Messenger.
TEST_F(TabletServerTest, ServerHostname) {
string server_hostname;
diff --git a/src/kudu/util/metrics-test.cc b/src/kudu/util/metrics-test.cc
index 4cb0b4c1c..326c30ec8 100644
--- a/src/kudu/util/metrics-test.cc
+++ b/src/kudu/util/metrics-test.cc
@@ -1303,6 +1303,10 @@ METRIC_DEFINE_counter(test_entity, warn_counter, "Warn Metric", MetricUnit::kReq
"Description of warn metric",
kudu::MetricLevel::kWarn);
+METRIC_DEFINE_counter(test_entity, info_counter, "Info Metric", MetricUnit::kRequests,
+ "Description of info metric",
+ kudu::MetricLevel::kInfo);
+
METRIC_DEFINE_counter(test_entity, debug_counter, "Debug Metric", MetricUnit::kRequests,
"Description of debug metric",
kudu::MetricLevel::kDebug);
@@ -1617,4 +1621,193 @@ TEST_F(MetricsTest, PrometheusEmptyHostnameLabel) {
"kudu_tablet_test_counter{type=\"tablet\",id=\"t1\",unit_type=\"bytes\"} 1\n");
}
+// Tests for Prometheus-format filtering via MetricPrometheusOptions::filters.
+// Tests that use real entity types (tablet, table, server) enable the new
+// entity-labels format for readable assertions. PrometheusFilterByEntityLevel
+// is the exception: it uses test_entity (not a known type) and therefore
+// stays in legacy format.
+// TODO(KUDU-3775): once BuildPrometheusLabels() handles arbitrary entity types,
+// switch PrometheusFilterByEntityLevel to entity-labels format too.
+
+TEST_F(MetricsTest, PrometheusFilterByEntityLevel) {
+ // test_entity is not a known Prometheus entity type, so BuildPrometheusLabels()
+ // would DCHECK with the entity-labels format. Pin legacy format explicitly so
+ // this test is not sensitive to the default value of the flag.
+ google::FlagSaver saver;
+ FLAGS_metrics_prometheus_use_entity_labels = false;
+
+ MetricRegistry registry;
+ auto entity = METRIC_ENTITY_test_entity.Instantiate(®istry, "level-test");
+ auto warn_m = METRIC_warn_counter.Instantiate(entity);
+ auto info_m = METRIC_info_counter.Instantiate(entity);
+ auto debug_m = METRIC_debug_counter.Instantiate(entity);
+
+ {
+ // "warn" level: only warn metric should appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_level = "warn";
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_CONTAINS(out.str(), "warn_counter");
+ ASSERT_STR_NOT_CONTAINS(out.str(), "info_counter");
+ ASSERT_STR_NOT_CONTAINS(out.str(), "debug_counter");
+ }
+ {
+ // "info" level: warn and info metrics should appear, debug should not.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_level = "info";
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_CONTAINS(out.str(), "warn_counter");
+ ASSERT_STR_CONTAINS(out.str(), "info_counter");
+ ASSERT_STR_NOT_CONTAINS(out.str(), "debug_counter");
+ }
+ {
+ // "debug" level: all three metrics should appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_level = "debug";
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_CONTAINS(out.str(), "warn_counter");
+ ASSERT_STR_CONTAINS(out.str(), "info_counter");
+ ASSERT_STR_CONTAINS(out.str(), "debug_counter");
+ }
+}
+
+TEST_F(MetricsTest, PrometheusFilterByEntityType) {
+ google::FlagSaver saver;
+ FLAGS_metrics_prometheus_use_entity_labels = true;
+
+ MetricRegistry registry;
+ auto tablet_entity = METRIC_ENTITY_tablet.Instantiate(®istry, "t1");
+ METRIC_tablet_test_counter.Instantiate(tablet_entity);
+
+ auto table_entity = METRIC_ENTITY_table.Instantiate(®istry, "tbl1");
+ METRIC_table_test_counter.Instantiate(table_entity);
+
+ {
+ // Filter to "tablet" type only: table counter must not appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_types = { "tablet" };
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_CONTAINS(out.str(), "type=\"tablet\"");
+ ASSERT_STR_NOT_CONTAINS(out.str(), "type=\"table\"");
+ }
+ // TODO(KUDU-3774): add a "table" filter case here that asserts only
+ // type="table" appears and type="tablet" does not. Currently blocked because
+ // MatchName() does substring matching, so "table" also matches "tablet".
+ {
+ // Non-existent type: no metrics should appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_types = { "nonexistent_type" };
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_NOT_CONTAINS(out.str(), "type=\"tablet\"");
+ ASSERT_STR_NOT_CONTAINS(out.str(), "type=\"table\"");
+ }
+}
+
+TEST_F(MetricsTest, PrometheusFilterByEntityId) {
+ google::FlagSaver saver;
+ FLAGS_metrics_prometheus_use_entity_labels = true;
+
+ MetricRegistry registry;
+ auto e_keep = METRIC_ENTITY_tablet.Instantiate(®istry, "tablet-keep");
+ auto e_drop = METRIC_ENTITY_tablet.Instantiate(®istry, "tablet-drop");
+ METRIC_tablet_test_counter.Instantiate(e_keep);
+ METRIC_tablet_test_counter.Instantiate(e_drop);
+
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_ids = { "tablet-keep" };
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ const auto& str = out.str();
+ ASSERT_STR_CONTAINS(str, "id=\"tablet-keep\"");
+ ASSERT_STR_NOT_CONTAINS(str, "id=\"tablet-drop\"");
+}
+
+TEST_F(MetricsTest, PrometheusFilterByMetricName) {
+ google::FlagSaver saver;
+ FLAGS_metrics_prometheus_use_entity_labels = true;
+
+ MetricRegistry registry;
+ auto entity = METRIC_ENTITY_tablet.Instantiate(®istry, "t1");
+ METRIC_tablet_test_counter.Instantiate(entity);
+
+ {
+ // Matching substring: the metric should appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_metrics = { "tablet_test_counter" };
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_CONTAINS(out.str(), "kudu_tablet_test_counter");
+ }
+ {
+ // Non-matching substring: no metrics should appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_metrics = { "nonexistent_metric" };
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ ASSERT_STR_NOT_CONTAINS(out.str(), "kudu_tablet_test_counter");
+ }
+}
+
+TEST_F(MetricsTest, PrometheusFilterByEntityAttributes) {
+ google::FlagSaver saver;
+ FLAGS_metrics_prometheus_use_entity_labels = true;
+
+ MetricRegistry registry;
+
+ // Two tablet entities belonging to different tables.
+ auto e_a = METRIC_ENTITY_tablet.Instantiate(
+ ®istry, "tablet-a", {{"table_name", "table_a"}});
+ auto e_b = METRIC_ENTITY_tablet.Instantiate(
+ ®istry, "tablet-b", {{"table_name", "table_b"}});
+ METRIC_tablet_test_counter.Instantiate(e_a);
+ METRIC_tablet_test_counter.Instantiate(e_b);
+
+ // Filter to table_a: only tablet-a should appear.
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_attrs = { "table_name", "table_a" };
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ const auto& str = out.str();
+ ASSERT_STR_CONTAINS(str, "id=\"tablet-a\"");
+ ASSERT_STR_NOT_CONTAINS(str, "id=\"tablet-b\"");
+}
+
+// Unrecognized ?level= values (e.g. a typo) fall through to the default
+// "debug" behaviour so that all metrics are included rather than silently
+// dropping everything.
+TEST_F(MetricsTest, PrometheusFilterUnrecognizedLevel) {
+ google::FlagSaver saver;
+ FLAGS_metrics_prometheus_use_entity_labels = false;
+
+ MetricRegistry registry;
+ auto entity = METRIC_ENTITY_test_entity.Instantiate(®istry, "level-test");
+ METRIC_warn_counter.Instantiate(entity);
+ METRIC_info_counter.Instantiate(entity);
+ METRIC_debug_counter.Instantiate(entity);
+
+ ostringstream out;
+ PrometheusWriter writer(&out);
+ MetricPrometheusOptions opts;
+ opts.filters.entity_level = "not_a_real_level";
+ ASSERT_OK(registry.WriteAsPrometheus(&writer, opts));
+ const auto& str = out.str();
+ ASSERT_STR_CONTAINS(str, "warn_counter");
+ ASSERT_STR_CONTAINS(str, "info_counter");
+ ASSERT_STR_CONTAINS(str, "debug_counter");
+}
+
} // namespace kudu
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index be227c499..6901ec50a 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -516,10 +516,9 @@ struct MetricJsonOptions {
// metrics in the Prometheus format.
struct MetricPrometheusOptions {
// Metrics are filtered per information in the 'filters' field,
- // see the in-line MetricFilters's documentation for more details.
- //
- // NOTE: for Prometheus format metrics, the only applicable filtering
- // is by 'MetricFilters::entity_level' field.
+ // see the MetricFilters documentation for details. All filtering
+ // dimensions (entity type, entity ID, entity attributes, metric name
+ // substrings, and severity level) are applicable.
MetricFilters filters;
// The hostname of the node serving the metrics. Attached as hostname="..."
diff --git a/src/kudu/util/prometheus_writer.cc b/src/kudu/util/prometheus_writer.cc
index fa1fb1378..583cd8ceb 100644
--- a/src/kudu/util/prometheus_writer.cc
+++ b/src/kudu/util/prometheus_writer.cc
@@ -58,6 +58,10 @@ string BuildPrometheusLabels(
const string& entity_type,
const string& entity_id,
const unordered_map<string, string>& attrs) {
+ // TODO(KUDU-3775): relax this to handle arbitrary entity types gracefully
+ // so that synthetic test entities (e.g. "test_entity") and any future
+ // entity types can use the new label-based Prometheus format without
+ // hitting this assertion.
DCHECK(entity_type == "server" ||
entity_type == "table" ||
entity_type == "tablet")
diff --git a/src/kudu/util/test_util.cc b/src/kudu/util/test_util.cc
index da95f0d90..b808b720f 100644
--- a/src/kudu/util/test_util.cc
+++ b/src/kudu/util/test_util.cc
@@ -771,6 +771,14 @@ void CheckPrometheusOutput(const string& prometheus_output) {
}
}
+void CheckNoPrometheusValueLines(const string& prometheus_output) {
+ vector<string> lines = strings::Split(prometheus_output, "\n", strings::SkipEmpty());
+ for (const auto& line : lines) {
+ ASSERT_TRUE(HasPrefixString(line, "#"))
+ << "Unexpected metric value line: " << line;
+ }
+}
+
std::ostream& operator<<(std::ostream& os, const IPMode& mode) {
os << IPModeToString(mode);
return os;
diff --git a/src/kudu/util/test_util.h b/src/kudu/util/test_util.h
index 3b141c9e0..6f8f33720 100644
--- a/src/kudu/util/test_util.h
+++ b/src/kudu/util/test_util.h
@@ -221,6 +221,11 @@ const std::unordered_map<std::string, std::string>& GetMasterWebserverEndpoints(
// and the master-test, so it is placed here.
void CheckPrometheusOutput(const std::string& prometheus_output);
+// Asserts that 'prometheus_output' contains no metric value lines (only comment/metadata
+// lines starting with '#' are allowed). Use after CheckPrometheusOutput() when the
+// expectation is that a filter produces an effectively empty result.
+void CheckNoPrometheusValueLines(const std::string& prometheus_output);
+
// Beautifies test output if a test scenario fails.
std::ostream& operator<<(std::ostream& os, const IPMode& mode);