You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ac...@apache.org on 2025/06/25 16:31:39 UTC
(kudu) branch branch-1.18.x updated: KUDU-3671 severity level filtering for /metrics_prometheus
This is an automated email from the ASF dual-hosted git repository.
achennaka pushed a commit to branch branch-1.18.x
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/branch-1.18.x by this push:
new 80d55c8d2 KUDU-3671 severity level filtering for /metrics_prometheus
80d55c8d2 is described below
commit 80d55c8d227cf376e2139310dd3f9b033f351f0b
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Tue Jun 24 20:35:03 2025 -0700
KUDU-3671 severity level filtering for /metrics_prometheus
This changelist implements severity level filtering for the
/metrics_prometheus endpoint of the embedded Kudu webserver based
on the current setting for the --metrics_default_level flag. This
is now similar to the filtering for the /metrics endpoints, but the
severity level filtering for the /metrics_prometheus cannot be
overridden by query options as of this changelist. Basic test
scenarios for Kudu master and tablet server are included.
Change-Id: If1542476f06d575594e8b3636cab3681f1b6fdfa
Reviewed-on: http://gerrit.cloudera.org:8080/23093
Reviewed-by: Abhishek Chennaka <ac...@cloudera.com>
Tested-by: Alexey Serbin <al...@apache.org>
(cherry picked from commit f7c956859e2f49c4cf1caffa969c1777a7a5d81c)
Reviewed-on: http://gerrit.cloudera.org:8080/23095
Tested-by: Abhishek Chennaka <ac...@cloudera.com>
---
src/kudu/master/master-test.cc | 53 +++++++++++++++++++++++++++++++-
src/kudu/server/default_path_handlers.cc | 19 ++++++++----
src/kudu/tserver/tablet_server-test.cc | 48 ++++++++++++++++++++++++++++-
src/kudu/util/metrics-test.cc | 4 +--
src/kudu/util/metrics.cc | 16 ++++------
src/kudu/util/metrics.h | 19 ++++++++++--
6 files changed, 135 insertions(+), 24 deletions(-)
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 28f162a80..a0f85554f 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -150,6 +150,7 @@ DECLARE_int64(on_disk_size_for_testing);
DECLARE_string(ipki_private_key_password_cmd);
DECLARE_string(location_mapping_cmd);
DECLARE_string(log_filename);
+DECLARE_string(metrics_default_level);
DECLARE_string(tsk_private_key_password_cmd);
DECLARE_string(webserver_doc_root);
@@ -3850,7 +3851,57 @@ TEST_F(MasterTest, SmokeTestPrometheusMetrics) {
ASSERT_OK(c.FetchURL(Substitute("http://$0/metrics_prometheus",
mini_master_->bound_http_addr().ToString()),
&buf));
- CheckPrometheusOutput(buf.ToString());
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+
+ // Since the defaut severity level for metrics is 'debug', there should
+ // be metrics of 'debug', 'info', and 'warn' levels in the output.
+ ASSERT_STR_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+}
+
+// Verify that Prometheus format metrics for Kudu masters are output
+// in accordance with the severity level filtering controlled by
+// the --metrics_default_level flag. This isn't an exhaustive scenario
+// in the sense of checking for the presense or absense of _all_ the metrics
+// of a particular severity level, but rather a smoke test scenario
+// to check for the presence and absense of just a few metrics,
+// aiming to detect regressions.
+TEST_F(MasterTest, PrometheusMetricsLevelFiltering) {
+ constexpr char kTableName[] = "prom_metrics_level_filterting";
+ const Schema kTableSchema(
+ { ColumnSchema("key", INT32), ColumnSchema("v1", UINT64) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ const auto url = Substitute("http://$0/metrics_prometheus",
+ mini_master_->bound_http_addr().ToString());
+ {
+ FLAGS_metrics_default_level = "info";
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(url, &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ // There should be no metrics of 'debug' level in the output.
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ // There should be metrics of both 'info' and 'warn' levels.
+ ASSERT_STR_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");// level: warn
+ }
+ {
+ FLAGS_metrics_default_level = "warn";
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(url, &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ // There should be no metrics of 'debug' or 'info' level in the output.
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // level: info
+ // There should be metrics only of the 'warn' level.
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+ }
}
} // namespace master
diff --git a/src/kudu/server/default_path_handlers.cc b/src/kudu/server/default_path_handlers.cc
index e125d9658..fb1a4552c 100644
--- a/src/kudu/server/default_path_handlers.cc
+++ b/src/kudu/server/default_path_handlers.cc
@@ -18,6 +18,7 @@
#include "kudu/server/default_path_handlers.h"
#include <sys/stat.h>
+// IWYU pragma: no_include <bits/struct_stat.h>
#include <cstddef>
#include <cstdint>
@@ -31,6 +32,7 @@
#include <gflags/gflags.h>
#include <glog/logging.h>
+
#include "kudu/util/prometheus_writer.h"
#include "kudu/util/version_info.h"
#include "kudu/util/version_info.pb.h"
@@ -85,11 +87,12 @@ TAG_FLAG(web_log_bytes, advanced);
TAG_FLAG(web_log_bytes, runtime);
DEFINE_string(metrics_default_level, "debug",
- "The default severity level to use when filtering the metrics. "
- "Valid choices are 'debug', 'info', and 'warn'. "
- "The levels are ordered and lower levels include the levels above them. "
- "This value can be overridden by passing the level query parameter to the "
- "'/metrics' endpoint.");
+ "The default severity level to use when filtering the metrics, "
+ "both in JSON and Prometheus formats. Valid choices are 'debug', "
+ "'info', and 'warn'. The levels are ordered and lower levels "
+ "include the levels above them. This value can be overridden "
+ "by passing the level query parameter to the '/metrics' endpoint "
+ "when fetching metrics in JSON format.");
TAG_FLAG(metrics_default_level, advanced);
TAG_FLAG(metrics_default_level, runtime);
TAG_FLAG(metrics_default_level, evolving);
@@ -522,8 +525,12 @@ static void WriteMetricsAsJson(const MetricRegistry* const metrics,
static void WriteMetricsAsPrometheus(const MetricRegistry* const metrics,
const Webserver::WebRequest& /*req*/,
Webserver::PrerenderedWebResponse* resp) {
+ MetricPrometheusOptions opts;
+ opts.filters.entity_level = FLAGS_metrics_default_level;
+
PrometheusWriter writer(&resp->output);
- WARN_NOT_OK(metrics->WriteAsPrometheus(&writer), "Couldn't write Prometheus metrics over HTTP");
+ WARN_NOT_OK(metrics->WriteAsPrometheus(&writer, opts),
+ "couldn't write Prometheus metrics over HTTP");
}
void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry* const metrics) {
diff --git a/src/kudu/tserver/tablet_server-test.cc b/src/kudu/tserver/tablet_server-test.cc
index b58be42c1..e47b0ec9b 100644
--- a/src/kudu/tserver/tablet_server-test.cc
+++ b/src/kudu/tserver/tablet_server-test.cc
@@ -216,6 +216,7 @@ DECLARE_int32(workload_stats_metric_collection_interval_ms);
DECLARE_string(block_manager);
DECLARE_string(env_inject_eio_globs);
DECLARE_string(env_inject_full_globs);
+DECLARE_string(metrics_default_level);
DECLARE_string(webserver_doc_root);
DECLARE_uint32(tablet_apply_pool_overload_threshold_ms);
@@ -4366,7 +4367,52 @@ TEST_F(TabletServerTest, SmokeTestPrometheusMetrics) {
ASSERT_OK(c.FetchURL(Substitute("http://$0/metrics_prometheus",
mini_server_->bound_http_addr().ToString()),
&buf));
- CheckPrometheusOutput(buf.ToString());
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+
+ // Since the defaut severity level for metrics is 'debug', there should
+ // be metrics of 'debug', 'info', and 'warn' levels in the output.
+ ASSERT_STR_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+}
+
+// Verify that Prometheus format Kudu metrics for tablet servers are output
+// in accordance with the severity level filtering controlled by the
+// --metrics_default_level flag. This isn't an exhaustive scenario in the sense
+// of checking for the presense or the absense of _all_ the metrics
+// of a particular severity level, but rather a smoke test scenario
+// to check for the presence and absense of just a few metrics,
+// aiming to detect regressions.
+TEST_F(TabletServerTest, PrometheusMetricsLevelFiltering) {
+ const auto url = Substitute("http://$0/metrics_prometheus",
+ mini_server_->bound_http_addr().ToString());
+ {
+ FLAGS_metrics_default_level = "info";
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(url, &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ // There should be no metrics of 'debug' level in the output.
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ // There should be metrics of both 'info' and 'warn' levels.
+ ASSERT_STR_MATCHES(str, "threads_running "); // level: info
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");// level: warn
+ }
+ {
+ FLAGS_metrics_default_level = "warn";
+ EasyCurl c;
+ faststring buf;
+ ASSERT_OK(c.FetchURL(url, &buf));
+ const auto& str = buf.ToString();
+ NO_FATALS(CheckPrometheusOutput(str));
+ // There should be no metrics of 'debug' or 'info' level in the output.
+ ASSERT_STR_NOT_MATCHES(str, "raft_term "); // level: debug
+ ASSERT_STR_NOT_MATCHES(str, "threads_running "); // level: info
+ // There should be metrics only of the 'warn' level.
+ ASSERT_STR_MATCHES(str, "rpcs_queue_overflow "); // level: warn
+ }
}
// Test that hostname is set properly for TabletServer's Messenger.
diff --git a/src/kudu/util/metrics-test.cc b/src/kudu/util/metrics-test.cc
index 259997871..c56f4a237 100644
--- a/src/kudu/util/metrics-test.cc
+++ b/src/kudu/util/metrics-test.cc
@@ -56,8 +56,6 @@ using std::vector;
DECLARE_int32(metrics_retirement_age_ms);
-DECLARE_string(metrics_default_level);
-
namespace kudu {
METRIC_DEFINE_entity(test_entity);
@@ -174,7 +172,7 @@ TEST_F(MetricsTest, TableAndTabletPrometheusTest) {
ostringstream output;
PrometheusWriter writer(&output);
- ASSERT_OK(registry_.WriteAsPrometheus(&writer));
+ ASSERT_OK(registry_.WriteAsPrometheus(&writer, {}));
// The order of elements in the output depends on the ordering in hash-map
// of metric entities and might be different in different STL implementations.
diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc
index 1715004af..b2b33c2ed 100644
--- a/src/kudu/util/metrics.cc
+++ b/src/kudu/util/metrics.cc
@@ -404,19 +404,14 @@ Status MetricEntity::WriteAsJson(JsonWriter* writer, const MetricJsonOptions& op
return Status::OK();
}
-Status MetricEntity::WriteAsPrometheus(PrometheusWriter* writer) const {
+Status MetricEntity::WriteAsPrometheus(
+ PrometheusWriter* writer, const MetricPrometheusOptions& opts) const {
static const string kIdMaster = "kudu.master";
static const string kIdTabletServer = "kudu.tabletserver";
- // Empty filters result in getting all the metrics for this MetricEntity.
- //
- // TODO(aserbin): instead of hard-coding, pass MetricFilters as a parameter
- MetricFilters filters;
- filters.entity_level = "debug";
-
MetricMap metrics;
AttributeMap attrs;
- const auto s = GetMetricsAndAttrs(filters, &metrics, &attrs);
+ const auto s = GetMetricsAndAttrs(opts.filters, &metrics, &attrs);
if (s.IsNotFound()) {
// Status::NotFound is returned when this entity has been filtered, treat it
// as OK, and skip printing it.
@@ -591,14 +586,15 @@ Status MetricRegistry::WriteAsJson(JsonWriter* writer, const MetricJsonOptions&
return Status::OK();
}
-Status MetricRegistry::WriteAsPrometheus(PrometheusWriter* writer) const {
+Status MetricRegistry::WriteAsPrometheus(
+ PrometheusWriter* writer, const MetricPrometheusOptions& opts) const {
EntityMap entities;
{
std::lock_guard l(lock_);
entities = entities_;
}
for (const auto& e : entities) {
- WARN_NOT_OK(e.second->WriteAsPrometheus(writer),
+ WARN_NOT_OK(e.second->WriteAsPrometheus(writer, opts),
Substitute("Failed to write entity $0 as Prometheus", e.second->id()));
}
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index 101b36740..9212a9039 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -512,6 +512,17 @@ struct MetricJsonOptions {
MetricMergeRules merge_rules;
};
+// Options to control the behavior of the metric end-points producing Kudu
+// metrics in the Prometheus format.
+struct MetricPrometheusOptions {
+ // Metrics are filtered per information in the 'filters' field,
+ // see the in-line MetricFilters's documentation for more details.
+ //
+ // NOTE: for Prometheus format metrics, the only applicable filtering
+ // is by 'MetricFilters::entity_level' field.
+ MetricFilters filters;
+};
+
class MetricEntityPrototype {
public:
explicit MetricEntityPrototype(const char* name);
@@ -667,7 +678,8 @@ class MetricEntity : public RefCountedThreadSafe<MetricEntity> {
// See MetricRegistry::WriteAsJson()
Status WriteAsJson(JsonWriter* writer, const MetricJsonOptions& opts) const;
- Status WriteAsPrometheus(PrometheusWriter* writer) const;
+ Status WriteAsPrometheus(PrometheusWriter* writer,
+ const MetricPrometheusOptions& opts) const;
// Collect metrics of this entity to 'collections'. Metrics will be filtered by 'filters',
// and will be merged under the rule of 'merge_rules'.
@@ -884,7 +896,8 @@ class MetricRegistry {
Status WriteAsJson(JsonWriter* writer, const MetricJsonOptions& opts) const;
// Writes metrics in this registry to given Prometheus 'writer'.
- Status WriteAsPrometheus(PrometheusWriter* writer) const;
+ Status WriteAsPrometheus(PrometheusWriter* writer,
+ const MetricPrometheusOptions& opts) const;
// For each registered entity, retires orphaned metrics. If an entity has no more
// metrics and there are no external references, entities are removed as well.
//
@@ -898,7 +911,7 @@ class MetricRegistry {
}
private:
- typedef std::unordered_map<std::string, scoped_refptr<MetricEntity> > EntityMap;
+ typedef std::unordered_map<std::string, scoped_refptr<MetricEntity>> EntityMap;
EntityMap entities_;
mutable simple_spinlock lock_;