You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ac...@apache.org on 2025/06/25 16:31:39 UTC

(kudu) branch branch-1.18.x updated: KUDU-3671 severity level filtering for /metrics_prometheus

This is an automated email from the ASF dual-hosted git repository.

achennaka pushed a commit to branch branch-1.18.x
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/branch-1.18.x by this push:
     new 80d55c8d2 KUDU-3671 severity level filtering for /metrics_prometheus
80d55c8d2 is described below

commit 80d55c8d227cf376e2139310dd3f9b033f351f0b
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Tue Jun 24 20:35:03 2025 -0700

    KUDU-3671 severity level filtering for /metrics_prometheus
    
    This changelist implements severity level filtering for the
    /metrics_prometheus endpoint of the embedded Kudu webserver based
    on the current setting for the --metrics_default_level flag.  This
    is now similar to the filtering for the /metrics endpoints, but the
    severity level filtering for the /metrics_prometheus cannot be
    overridden by query options as of this changelist.  Basic test
    scenarios for Kudu master and tablet server are included.
    
    Change-Id: If1542476f06d575594e8b3636cab3681f1b6fdfa
    Reviewed-on: http://gerrit.cloudera.org:8080/23093
    Reviewed-by: Abhishek Chennaka <ac...@cloudera.com>
    Tested-by: Alexey Serbin <al...@apache.org>
    (cherry picked from commit f7c956859e2f49c4cf1caffa969c1777a7a5d81c)
    Reviewed-on: http://gerrit.cloudera.org:8080/23095
    Tested-by: Abhishek Chennaka <ac...@cloudera.com>
---
 src/kudu/master/master-test.cc           | 53 +++++++++++++++++++++++++++++++-
 src/kudu/server/default_path_handlers.cc | 19 ++++++++----
 src/kudu/tserver/tablet_server-test.cc   | 48 ++++++++++++++++++++++++++++-
 src/kudu/util/metrics-test.cc            |  4 +--
 src/kudu/util/metrics.cc                 | 16 ++++------
 src/kudu/util/metrics.h                  | 19 ++++++++++--
 6 files changed, 135 insertions(+), 24 deletions(-)

diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 28f162a80..a0f85554f 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -150,6 +150,7 @@ DECLARE_int64(on_disk_size_for_testing);
 DECLARE_string(ipki_private_key_password_cmd);
 DECLARE_string(location_mapping_cmd);
 DECLARE_string(log_filename);
+DECLARE_string(metrics_default_level);
 DECLARE_string(tsk_private_key_password_cmd);
 DECLARE_string(webserver_doc_root);
 
@@ -3850,7 +3851,57 @@ TEST_F(MasterTest, SmokeTestPrometheusMetrics) {
   ASSERT_OK(c.FetchURL(Substitute("http://$0/metrics_prometheus",
                                   mini_master_->bound_http_addr().ToString()),
                        &buf));
-  CheckPrometheusOutput(buf.ToString());
+  const auto& str = buf.ToString();
+  NO_FATALS(CheckPrometheusOutput(str));
+
+  // Since the defaut severity level for metrics is 'debug', there should
+  // be metrics of 'debug', 'info', and 'warn' levels in the output.
+  ASSERT_STR_MATCHES(str, "raft_term ");            // level: debug
+  ASSERT_STR_MATCHES(str, "threads_running ");      // level: info
+  ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");  // level: warn
+}
+
+// Verify that Prometheus format metrics for Kudu masters are output
+// in accordance with the severity level filtering controlled by
+// the --metrics_default_level flag. This isn't an exhaustive scenario
+// in the sense of checking for the presense or absense of _all_ the metrics
+// of a particular severity level, but rather a smoke test scenario
+// to check for the presence and absense of just a few metrics,
+// aiming to detect regressions.
+TEST_F(MasterTest, PrometheusMetricsLevelFiltering) {
+  constexpr char kTableName[] = "prom_metrics_level_filterting";
+  const Schema kTableSchema(
+      { ColumnSchema("key", INT32), ColumnSchema("v1", UINT64) }, 1);
+  ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+  const auto url = Substitute("http://$0/metrics_prometheus",
+                              mini_master_->bound_http_addr().ToString());
+  {
+    FLAGS_metrics_default_level = "info";
+    EasyCurl c;
+    faststring buf;
+    ASSERT_OK(c.FetchURL(url, &buf));
+    const auto& str = buf.ToString();
+    NO_FATALS(CheckPrometheusOutput(str));
+    // There should be no metrics of 'debug' level in the output.
+    ASSERT_STR_NOT_MATCHES(str, "raft_term ");      // level: debug
+    // There should be metrics of both 'info' and 'warn' levels.
+    ASSERT_STR_MATCHES(str, "threads_running ");    // level: info
+    ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");// level: warn
+  }
+  {
+    FLAGS_metrics_default_level = "warn";
+    EasyCurl c;
+    faststring buf;
+    ASSERT_OK(c.FetchURL(url, &buf));
+    const auto& str = buf.ToString();
+    NO_FATALS(CheckPrometheusOutput(str));
+    // There should be no metrics of 'debug' or 'info' level in the output.
+    ASSERT_STR_NOT_MATCHES(str, "raft_term ");        // level: debug
+    ASSERT_STR_NOT_MATCHES(str, "threads_running ");  // level: info
+    // There should be metrics only of the 'warn' level.
+    ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");  // level: warn
+  }
 }
 
 } // namespace master
diff --git a/src/kudu/server/default_path_handlers.cc b/src/kudu/server/default_path_handlers.cc
index e125d9658..fb1a4552c 100644
--- a/src/kudu/server/default_path_handlers.cc
+++ b/src/kudu/server/default_path_handlers.cc
@@ -18,6 +18,7 @@
 #include "kudu/server/default_path_handlers.h"
 
 #include <sys/stat.h>
+// IWYU pragma: no_include <bits/struct_stat.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -31,6 +32,7 @@
 
 #include <gflags/gflags.h>
 #include <glog/logging.h>
+
 #include "kudu/util/prometheus_writer.h"
 #include "kudu/util/version_info.h"
 #include "kudu/util/version_info.pb.h"
@@ -85,11 +87,12 @@ TAG_FLAG(web_log_bytes, advanced);
 TAG_FLAG(web_log_bytes, runtime);
 
 DEFINE_string(metrics_default_level, "debug",
-              "The default severity level to use when filtering the metrics. "
-              "Valid choices are 'debug', 'info', and 'warn'. "
-              "The levels are ordered and lower levels include the levels above them. "
-              "This value can be overridden by passing the level query parameter to the "
-              "'/metrics' endpoint.");
+              "The default severity level to use when filtering the metrics, "
+              "both in JSON and Prometheus formats. Valid choices are 'debug', "
+              "'info', and 'warn'. The levels are ordered and lower levels "
+              "include the levels above them. This value can be overridden "
+              "by passing the level query parameter to the '/metrics' endpoint "
+              "when fetching metrics in JSON format.");
 TAG_FLAG(metrics_default_level, advanced);
 TAG_FLAG(metrics_default_level, runtime);
 TAG_FLAG(metrics_default_level, evolving);
@@ -522,8 +525,12 @@ static void WriteMetricsAsJson(const MetricRegistry* const metrics,
 static void WriteMetricsAsPrometheus(const MetricRegistry* const metrics,
                                      const Webserver::WebRequest& /*req*/,
                                      Webserver::PrerenderedWebResponse* resp) {
+  MetricPrometheusOptions opts;
+  opts.filters.entity_level = FLAGS_metrics_default_level;
+
   PrometheusWriter writer(&resp->output);
-  WARN_NOT_OK(metrics->WriteAsPrometheus(&writer), "Couldn't write Prometheus metrics over HTTP");
+  WARN_NOT_OK(metrics->WriteAsPrometheus(&writer, opts),
+              "couldn't write Prometheus metrics over HTTP");
 }
 
 void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry* const metrics) {
diff --git a/src/kudu/tserver/tablet_server-test.cc b/src/kudu/tserver/tablet_server-test.cc
index b58be42c1..e47b0ec9b 100644
--- a/src/kudu/tserver/tablet_server-test.cc
+++ b/src/kudu/tserver/tablet_server-test.cc
@@ -216,6 +216,7 @@ DECLARE_int32(workload_stats_metric_collection_interval_ms);
 DECLARE_string(block_manager);
 DECLARE_string(env_inject_eio_globs);
 DECLARE_string(env_inject_full_globs);
+DECLARE_string(metrics_default_level);
 DECLARE_string(webserver_doc_root);
 DECLARE_uint32(tablet_apply_pool_overload_threshold_ms);
 
@@ -4366,7 +4367,52 @@ TEST_F(TabletServerTest, SmokeTestPrometheusMetrics) {
   ASSERT_OK(c.FetchURL(Substitute("http://$0/metrics_prometheus",
                                   mini_server_->bound_http_addr().ToString()),
                        &buf));
-  CheckPrometheusOutput(buf.ToString());
+  const auto& str = buf.ToString();
+  NO_FATALS(CheckPrometheusOutput(str));
+
+  // Since the defaut severity level for metrics is 'debug', there should
+  // be metrics of 'debug', 'info', and 'warn' levels in the output.
+  ASSERT_STR_MATCHES(str, "raft_term ");            // level: debug
+  ASSERT_STR_MATCHES(str, "threads_running ");      // level: info
+  ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");  // level: warn
+}
+
+// Verify that Prometheus format Kudu metrics for tablet servers are output
+// in accordance with the severity level filtering controlled by the
+// --metrics_default_level flag. This isn't an exhaustive scenario in the sense
+// of checking for the presense or the absense of _all_ the metrics
+// of a particular severity level, but rather a smoke test scenario
+// to check for the presence and absense of just a few metrics,
+// aiming to detect regressions.
+TEST_F(TabletServerTest, PrometheusMetricsLevelFiltering) {
+  const auto url = Substitute("http://$0/metrics_prometheus",
+                              mini_server_->bound_http_addr().ToString());
+  {
+    FLAGS_metrics_default_level = "info";
+    EasyCurl c;
+    faststring buf;
+    ASSERT_OK(c.FetchURL(url, &buf));
+    const auto& str = buf.ToString();
+    NO_FATALS(CheckPrometheusOutput(str));
+    // There should be no metrics of 'debug' level in the output.
+    ASSERT_STR_NOT_MATCHES(str, "raft_term ");      // level: debug
+    // There should be metrics of both 'info' and 'warn' levels.
+    ASSERT_STR_MATCHES(str, "threads_running ");    // level: info
+    ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");// level: warn
+  }
+  {
+    FLAGS_metrics_default_level = "warn";
+    EasyCurl c;
+    faststring buf;
+    ASSERT_OK(c.FetchURL(url, &buf));
+    const auto& str = buf.ToString();
+    NO_FATALS(CheckPrometheusOutput(str));
+    // There should be no metrics of 'debug' or 'info' level in the output.
+    ASSERT_STR_NOT_MATCHES(str, "raft_term ");        // level: debug
+    ASSERT_STR_NOT_MATCHES(str, "threads_running ");  // level: info
+    // There should be metrics only of the 'warn' level.
+    ASSERT_STR_MATCHES(str, "rpcs_queue_overflow ");  // level: warn
+  }
 }
 
 // Test that hostname is set properly for TabletServer's Messenger.
diff --git a/src/kudu/util/metrics-test.cc b/src/kudu/util/metrics-test.cc
index 259997871..c56f4a237 100644
--- a/src/kudu/util/metrics-test.cc
+++ b/src/kudu/util/metrics-test.cc
@@ -56,8 +56,6 @@ using std::vector;
 
 DECLARE_int32(metrics_retirement_age_ms);
 
-DECLARE_string(metrics_default_level);
-
 namespace kudu {
 
 METRIC_DEFINE_entity(test_entity);
@@ -174,7 +172,7 @@ TEST_F(MetricsTest, TableAndTabletPrometheusTest) {
 
   ostringstream output;
   PrometheusWriter writer(&output);
-  ASSERT_OK(registry_.WriteAsPrometheus(&writer));
+  ASSERT_OK(registry_.WriteAsPrometheus(&writer, {}));
 
   // The order of elements in the output depends on the ordering in hash-map
   // of metric entities and might be different in different STL implementations.
diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc
index 1715004af..b2b33c2ed 100644
--- a/src/kudu/util/metrics.cc
+++ b/src/kudu/util/metrics.cc
@@ -404,19 +404,14 @@ Status MetricEntity::WriteAsJson(JsonWriter* writer, const MetricJsonOptions& op
   return Status::OK();
 }
 
-Status MetricEntity::WriteAsPrometheus(PrometheusWriter* writer) const {
+Status MetricEntity::WriteAsPrometheus(
+    PrometheusWriter* writer, const MetricPrometheusOptions& opts) const {
   static const string kIdMaster = "kudu.master";
   static const string kIdTabletServer = "kudu.tabletserver";
 
-  // Empty filters result in getting all the metrics for this MetricEntity.
-  //
-  // TODO(aserbin): instead of hard-coding, pass MetricFilters as a parameter
-  MetricFilters filters;
-  filters.entity_level = "debug";
-
   MetricMap metrics;
   AttributeMap attrs;
-  const auto s = GetMetricsAndAttrs(filters, &metrics, &attrs);
+  const auto s = GetMetricsAndAttrs(opts.filters, &metrics, &attrs);
   if (s.IsNotFound()) {
     // Status::NotFound is returned when this entity has been filtered, treat it
     // as OK, and skip printing it.
@@ -591,14 +586,15 @@ Status MetricRegistry::WriteAsJson(JsonWriter* writer, const MetricJsonOptions&
   return Status::OK();
 }
 
-Status MetricRegistry::WriteAsPrometheus(PrometheusWriter* writer) const {
+Status MetricRegistry::WriteAsPrometheus(
+    PrometheusWriter* writer, const MetricPrometheusOptions& opts) const {
   EntityMap entities;
   {
     std::lock_guard l(lock_);
     entities = entities_;
   }
   for (const auto& e : entities) {
-    WARN_NOT_OK(e.second->WriteAsPrometheus(writer),
+    WARN_NOT_OK(e.second->WriteAsPrometheus(writer, opts),
                 Substitute("Failed to write entity $0 as Prometheus", e.second->id()));
   }
 
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index 101b36740..9212a9039 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -512,6 +512,17 @@ struct MetricJsonOptions {
   MetricMergeRules merge_rules;
 };
 
+// Options to control the behavior of the metric end-points producing Kudu
+// metrics in the Prometheus format.
+struct MetricPrometheusOptions {
+  // Metrics are filtered per information in the 'filters' field,
+  // see the in-line MetricFilters's documentation for more details.
+  //
+  // NOTE: for Prometheus format metrics, the only applicable filtering
+  //       is by 'MetricFilters::entity_level' field.
+  MetricFilters filters;
+};
+
 class MetricEntityPrototype {
  public:
   explicit MetricEntityPrototype(const char* name);
@@ -667,7 +678,8 @@ class MetricEntity : public RefCountedThreadSafe<MetricEntity> {
   // See MetricRegistry::WriteAsJson()
   Status WriteAsJson(JsonWriter* writer, const MetricJsonOptions& opts) const;
 
-  Status WriteAsPrometheus(PrometheusWriter* writer) const;
+  Status WriteAsPrometheus(PrometheusWriter* writer,
+                           const MetricPrometheusOptions& opts) const;
 
   // Collect metrics of this entity to 'collections'. Metrics will be filtered by 'filters',
   // and will be merged under the rule of 'merge_rules'.
@@ -884,7 +896,8 @@ class MetricRegistry {
   Status WriteAsJson(JsonWriter* writer, const MetricJsonOptions& opts) const;
 
   // Writes metrics in this registry to given Prometheus 'writer'.
-  Status WriteAsPrometheus(PrometheusWriter* writer) const;
+  Status WriteAsPrometheus(PrometheusWriter* writer,
+                           const MetricPrometheusOptions& opts) const;
   // For each registered entity, retires orphaned metrics. If an entity has no more
   // metrics and there are no external references, entities are removed as well.
   //
@@ -898,7 +911,7 @@ class MetricRegistry {
   }
 
  private:
-  typedef std::unordered_map<std::string, scoped_refptr<MetricEntity> > EntityMap;
+  typedef std::unordered_map<std::string, scoped_refptr<MetricEntity>> EntityMap;
   EntityMap entities_;
 
   mutable simple_spinlock lock_;