You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2026/05/14 11:50:34 UTC
(kudu) branch master updated: KUDU-3693: Add bearer token auth for Prometheus
This is an automated email from the ASF dual-hosted git repository.
martongreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 8882db7bd KUDU-3693: Add bearer token auth for Prometheus
8882db7bd is described below
commit 8882db7bd58d6fbdfd06a85b979a44fe4bce14d9
Author: Marton Greber <gr...@gmail.com>
AuthorDate: Wed May 6 17:14:40 2026 +0200
KUDU-3693: Add bearer token auth for Prometheus
Prometheus does not support SPNEGO, so scraping a secured Kudu cluster
requires an alternative authentication mechanism. Add
--webserver_prometheus_token_cmd: when set, designated Prometheus
endpoints (/metrics_prometheus, /prometheus-sd) accept an
"Authorization: Bearer <token>" header as an alternative to SPNEGO.
This allows Prometheus scrapers to authenticate on an otherwise
SPNEGO-secured cluster. Clients that do support SPNEGO (e.g.
curl --negotiate, Knox Gateway) can continue to use it on these
endpoints as before.
Adds unit tests in webserver-test.cc and integration tests in
prometheus_auth-itest.cc covering correct, wrong, and missing
token scenarios for both scrape and SD endpoints.
Example Prometheus configs for using the Bearer token:
Static scrape config:
global:
scrape_configs:
- job_name: kudu
metrics_path: /metrics_prometheus
scrape_interval: 2s
authorization:
type: Bearer
credentials: <webserver_prometheus_token>
static_configs:
- targets:
- 127.0.0.1:8050
- 127.0.0.1:8051
- 127.0.0.1:8052
SD scrape config:
global:
scrape_configs:
- job_name: kudu
metrics_path: /metrics_prometheus
scrape_interval: 2s
authorization:
type: Bearer
credentials: <webserver_prometheus_token>
http_sd_configs:
- url: http://127.0.0.1:8050/prometheus-sd
refresh_interval: 2s
authorization:
type: Bearer
credentials: <webserver_prometheus_token>
- url: http://127.0.0.1:8051/prometheus-sd
refresh_interval: 2s
authorization:
type: Bearer
credentials: <webserver_prometheus_token>
- url: http://127.0.0.1:8052/prometheus-sd
refresh_interval: 2s
authorization:
type: Bearer
credentials: <webserver_prometheus_token>
Change-Id: If8169196a2c49d64865095f3bc6da1d3bfbcecfb
Reviewed-on: http://gerrit.cloudera.org:8080/24278
Tested-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Zoltan Chovan <zc...@cloudera.com>
Reviewed-by: Gabriella Lotz <lo...@gmail.com>
---
src/kudu/integration-tests/CMakeLists.txt | 1 +
.../integration-tests/prometheus_auth-itest.cc | 271 +++++++++++++++++++++
src/kudu/master/master_path_handlers.cc | 1 +
src/kudu/server/default_path_handlers.cc | 1 +
src/kudu/server/webserver-test.cc | 175 +++++++++++++
src/kudu/server/webserver.cc | 181 ++++++++++----
src/kudu/server/webserver.h | 17 +-
src/kudu/server/webserver_options.cc | 26 ++
src/kudu/server/webserver_options.h | 1 +
src/kudu/util/mini_prometheus.cc | 42 +++-
src/kudu/util/mini_prometheus.h | 19 ++
11 files changed, 687 insertions(+), 48 deletions(-)
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index a0feb8118..9009b192f 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -93,6 +93,7 @@ ADD_KUDU_TEST(linked_list-test RUN_SERIAL true)
ADD_KUDU_TEST(log-rolling-itest)
ADD_KUDU_TEST(maintenance_mode-itest NUM_SHARDS 8 PROCESSORS 4
DATA_FILES ../scripts/assign-location.py)
+ADD_KUDU_TEST(prometheus_auth-itest)
ADD_KUDU_TEST(prometheus_sd-itest
DATA_FILES ../scripts/assign-location.py)
ADD_KUDU_TEST(master_authz-itest RUN_SERIAL true NUM_SHARDS 8 PROCESSORS 4)
diff --git a/src/kudu/integration-tests/prometheus_auth-itest.cc b/src/kudu/integration-tests/prometheus_auth-itest.cc
new file mode 100644
index 000000000..271a883c9
--- /dev/null
+++ b/src/kudu/integration-tests/prometheus_auth-itest.cc
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <rapidjson/document.h>
+
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/security/test/test_certs.h"
+#include "kudu/util/mini_prometheus.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+
+using cluster::ExternalMiniCluster;
+using cluster::ExternalMiniClusterOptions;
+
+constexpr const char* kToken = "test-prometheus-bearer-token";
+constexpr int kScrapeIntervalSecs = 2;
+const MonoDelta kScrapeTimeout = MonoDelta::FromSeconds(120);
+
+// Base fixture for Prometheus authentication integration tests.
+// Starts an ExternalMiniCluster with Kerberos enabled (which automatically
+// sets --webserver_require_spnego=true on every daemon) and a TLS-enabled
+// webserver. No Prometheus bearer token is configured by default.
+class PrometheusSpnegoITest : public KuduTest {
+ public:
+ void SetUp() override {
+ KuduTest::SetUp();
+
+ // Generate one self-signed TLS cert shared by all daemons.
+ // Prometheus is configured with insecure_skip_verify=true because the
+ // ExternalMiniCluster binds each daemon to a dynamically-chosen loopback
+ // alias (e.g. 127.x.x.x) that does not match the SAN in the test cert,
+ // making hostname-based validation impossible. Encryption is still active;
+ // only the certificate identity check is skipped, mirroring the pattern
+ // used by webserver-crawl-itest.cc (curl.set_verify_peer(false)).
+ string cert_file, pk_file, pw;
+ ASSERT_OK(security::CreateTestSSLCertWithEncryptedKey(
+ GetTestDataDirectory(), &cert_file, &pk_file, &pw));
+
+ ExternalMiniClusterOptions opts;
+ opts.enable_kerberos = true;
+ opts.num_masters = 3;
+ opts.num_tablet_servers = 1;
+
+ vector<string> extra_flags = {
+ Substitute("--webserver_certificate_file=$0", cert_file),
+ Substitute("--webserver_private_key_file=$0", pk_file),
+ Substitute("--webserver_private_key_password_cmd=echo $0", pw),
+ };
+ AddExtraFlags(&extra_flags);
+ opts.extra_master_flags = extra_flags;
+ opts.extra_tserver_flags = extra_flags;
+
+ cluster_.reset(new ExternalMiniCluster(std::move(opts)));
+ ASSERT_OK(cluster_->Start());
+ }
+
+ void TearDown() override {
+ if (prom_) {
+ WARN_NOT_OK(prom_->Stop(), "Failed to stop MiniPrometheus");
+ prom_.reset();
+ }
+ if (cluster_) {
+ cluster_->Shutdown();
+ }
+ KuduTest::TearDown();
+ }
+
+ protected:
+ virtual void AddExtraFlags(vector<string>* /*flags*/) {}
+
+ int TotalDaemons() const {
+ return cluster_->num_masters() + cluster_->num_tablet_servers();
+ }
+
+ vector<string> StaticTargets() const {
+ vector<string> targets;
+ for (int i = 0; i < cluster_->num_masters(); ++i) {
+ targets.push_back(cluster_->master(i)->bound_http_hostport().ToString());
+ }
+ for (int i = 0; i < cluster_->num_tablet_servers(); ++i) {
+ targets.push_back(cluster_->tablet_server(i)->bound_http_hostport().ToString());
+ }
+ return targets;
+ }
+
+ vector<string> MasterSDUrls() const {
+ vector<string> urls;
+ for (int i = 0; i < cluster_->num_masters(); ++i) {
+ urls.push_back(Substitute("https://$0/prometheus-sd",
+ cluster_->master(i)->bound_http_hostport().ToString()));
+ }
+ return urls;
+ }
+
+ MiniPrometheusOptions BasePromOpts() const {
+ MiniPrometheusOptions opts;
+ opts.scheme = "https";
+ opts.skip_tls_verify = true;
+ opts.scrape_interval = Substitute("$0s", kScrapeIntervalSecs);
+ opts.sd_refresh_interval = Substitute("$0s", kScrapeIntervalSecs);
+ return opts;
+ }
+
+ // Waits several SD refresh cycles then asserts no active targets exist.
+ void AssertNoTargetsDiscovered() {
+ SleepFor(MonoDelta::FromSeconds(kScrapeIntervalSecs * 3));
+ rapidjson::Document doc;
+ ASSERT_OK(prom_->GetTargets(&doc));
+ ASSERT_EQ(0, doc["data"]["activeTargets"].Size())
+ << "Expected no active targets when SD authentication fails";
+ }
+
+ unique_ptr<ExternalMiniCluster> cluster_;
+ unique_ptr<MiniPrometheus> prom_;
+};
+
+// With SPNEGO required cluster-wide but no bearer token configured, Prometheus
+// cannot authenticate against the scrape endpoint (gets 401 Negotiate) and all
+// static targets report health="down".
+TEST_F(PrometheusSpnegoITest, TestScrapeFailsWithSpnegoOnly) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.static_targets = StaticTargets();
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ ASSERT_OK(prom_->WaitForTargetHealth(TotalDaemons(), "down", kScrapeTimeout));
+}
+
+// With SPNEGO required cluster-wide but no bearer token configured, Prometheus
+// cannot authenticate against the SD endpoint (gets 401 Negotiate) and
+// discovers no targets at all.
+TEST_F(PrometheusSpnegoITest, TestSdDiscoveryFailsWithSpnegoOnly) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.master_sd_urls = MasterSDUrls();
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ NO_FATALS(AssertNoTargetsDiscovered());
+}
+
+// Extends PrometheusSpnegoITest by also configuring a Prometheus bearer token
+// on every daemon.
+class PrometheusTokenITest : public PrometheusSpnegoITest {
+ protected:
+ void AddExtraFlags(vector<string>* flags) override {
+ flags->push_back(Substitute("--webserver_prometheus_token_cmd=echo $0", kToken));
+ }
+};
+
+// When Prometheus is configured with the correct bearer token, all targets
+// are scraped successfully despite SPNEGO being required cluster-wide.
+TEST_F(PrometheusTokenITest, TestCorrectTokenScrapeSucceeds) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.static_targets = StaticTargets();
+ opts.bearer_token = kToken;
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ ASSERT_OK(prom_->WaitForActiveTargets(TotalDaemons(), kScrapeTimeout));
+}
+
+// When Prometheus is configured with the wrong bearer token, all targets
+// report health="down" due to HTTP 401 responses from the Kudu webservers.
+TEST_F(PrometheusTokenITest, TestWrongTokenScrapeFails) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.static_targets = StaticTargets();
+ opts.bearer_token = "wrong-token";
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ ASSERT_OK(prom_->WaitForTargetHealth(TotalDaemons(), "down", kScrapeTimeout));
+}
+
+// When Prometheus sends no bearer token, all targets report health="down"
+// because the Kudu webservers reject the unauthenticated request with HTTP 401.
+TEST_F(PrometheusTokenITest, TestMissingTokenScrapeFails) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.static_targets = StaticTargets();
+ // No bearer_token set.
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ ASSERT_OK(prom_->WaitForTargetHealth(TotalDaemons(), "down", kScrapeTimeout));
+}
+
+// With the correct bearer token configured for both SD and scrape, Prometheus
+// discovers all targets via the master SD endpoint and scrapes them
+// successfully, exercising the full SD -> scrape path with auth end-to-end.
+TEST_F(PrometheusTokenITest, TestCorrectTokenSdAndScrapeSucceed) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.master_sd_urls = MasterSDUrls();
+ opts.bearer_token = kToken;
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ ASSERT_OK(prom_->WaitForActiveTargets(TotalDaemons(), kScrapeTimeout));
+}
+
+// When the SD token is wrong, Prometheus cannot authenticate against the
+// /prometheus-sd endpoint (HTTP 401) and never discovers any targets.
+TEST_F(PrometheusTokenITest, TestWrongTokenSdFails) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.master_sd_urls = MasterSDUrls();
+ opts.bearer_token = "wrong-token";
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ NO_FATALS(AssertNoTargetsDiscovered());
+}
+
+// When no SD token is supplied, Prometheus cannot authenticate against the
+// /prometheus-sd endpoint (HTTP 401) and never discovers any targets.
+TEST_F(PrometheusTokenITest, TestMissingTokenSdFails) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+
+ MiniPrometheusOptions opts = BasePromOpts();
+ opts.master_sd_urls = MasterSDUrls();
+ // No bearer_token set.
+ prom_.reset(new MiniPrometheus(opts));
+ ASSERT_OK(prom_->Start());
+
+ NO_FATALS(AssertNoTargetsDiscovered());
+}
+
+} // namespace kudu
diff --git a/src/kudu/master/master_path_handlers.cc b/src/kudu/master/master_path_handlers.cc
index 16672b092..1375e0049 100644
--- a/src/kudu/master/master_path_handlers.cc
+++ b/src/kudu/master/master_path_handlers.cc
@@ -1090,6 +1090,7 @@ Status MasterPathHandlers::Register(Webserver* server) {
this->HandlePrometheusSD(req, resp);
},
false /*is_on_nav_bar*/);
+ server->MarkPathAsPrometheus("/prometheus-sd");
return Status::OK();
}
diff --git a/src/kudu/server/default_path_handlers.cc b/src/kudu/server/default_path_handlers.cc
index 5ff258dcb..a886a5504 100644
--- a/src/kudu/server/default_path_handlers.cc
+++ b/src/kudu/server/default_path_handlers.cc
@@ -579,6 +579,7 @@ void RegisterMetricsPrometheusHandler(Webserver* webserver,
constexpr bool is_on_nav_bar = true;
webserver->RegisterPrerenderedPathHandler("/metrics_prometheus", "Prometheus Metrics", callback,
StyleMode::UNSTYLED, is_on_nav_bar);
+ webserver->MarkPathAsPrometheus("/metrics_prometheus");
}
} // namespace kudu
diff --git a/src/kudu/server/webserver-test.cc b/src/kudu/server/webserver-test.cc
index be9e7706d..973f31d0d 100644
--- a/src/kudu/server/webserver-test.cc
+++ b/src/kudu/server/webserver-test.cc
@@ -18,6 +18,7 @@
#include "kudu/server/webserver.h"
#include <cstdlib>
+#include <cstring>
#include <functional>
#include <initializer_list>
#include <memory>
@@ -469,6 +470,180 @@ TEST_P(SpnegoWebserverTest, TestAuthNotRequiredForOptions) {
NO_FATALS(RunTestOptions());
}
+class PrometheusUnsecuredWebserverTest : public WebserverTest {
+ protected:
+ void SetUp() override {
+ WebserverTest::SetUp();
+ server_->RegisterPrerenderedPathHandler(
+ "/metrics_prometheus", "Prometheus Metrics",
+ [](const Webserver::WebRequest& /*req*/,
+ Webserver::PrerenderedWebResponse* resp) {
+ resp->output << "prometheus_metric 1";
+ },
+ StyleMode::UNSTYLED, /*is_on_nav_bar=*/false);
+ server_->MarkPathAsPrometheus("/metrics_prometheus");
+ }
+
+ string PrometheusUrl() const {
+ return Substitute("http://$0/metrics_prometheus", addr_.ToString());
+ }
+};
+
+INSTANTIATE_TEST_SUITE_P(Parameters, PrometheusUnsecuredWebserverTest,
+ testing::Values(IPMode::IPV4, IPMode::IPV6, IPMode::DUAL));
+
+// Without a prometheus_token configured, the Prometheus endpoint is freely
+// accessible with no Authorization header required.
+TEST_P(PrometheusUnsecuredWebserverTest, TestAccessibleWithoutToken) {
+ ASSERT_OK(curl_.FetchURL(PrometheusUrl(), &buf_));
+ ASSERT_STR_CONTAINS(buf_.ToString(), "prometheus_metric");
+}
+
+// Without a prometheus_token configured, sending a bearer token does not
+// cause any error — it is simply ignored.
+TEST_P(PrometheusUnsecuredWebserverTest, TestBearerTokenIgnoredWhenNoTokenConfigured) {
+ ASSERT_OK(curl_.FetchURL(PrometheusUrl(), &buf_,
+ {"Authorization: Bearer some-token"}));
+ ASSERT_STR_CONTAINS(buf_.ToString(), "prometheus_metric");
+}
+
+class PrometheusTokenWebserverTest : public SpnegoWebserverTest {
+ protected:
+ static constexpr const char* kPrometheusToken = "test-prometheus-token";
+
+ void MaybeSetupSpnego(WebserverOptions* opts) override {
+ SpnegoWebserverTest::MaybeSetupSpnego(opts);
+ opts->prometheus_token_cmd = Substitute("echo $0", kPrometheusToken);
+ }
+
+ // Register a stub /metrics_prometheus handler that the test can hit, then
+ // mark it as a Prometheus path so bearer token auth applies.
+ void SetUp() override {
+ SpnegoWebserverTest::SetUp();
+ server_->RegisterPrerenderedPathHandler(
+ "/metrics_prometheus", "Prometheus Metrics",
+ [](const Webserver::WebRequest& /*req*/,
+ Webserver::PrerenderedWebResponse* resp) {
+ resp->output << "prometheus_metric 1";
+ },
+ StyleMode::UNSTYLED, /*is_on_nav_bar=*/false);
+ server_->MarkPathAsPrometheus("/metrics_prometheus");
+ }
+
+ string PrometheusUrl() const {
+ return Substitute(use_ssl() ? "https://$0/metrics_prometheus"
+ : "http://$0/metrics_prometheus",
+ addr_.ToString());
+ }
+};
+
+INSTANTIATE_TEST_SUITE_P(Parameters, PrometheusTokenWebserverTest,
+ testing::Values(IPMode::IPV4, IPMode::IPV6, IPMode::DUAL));
+
+// No Authorization header on a Prometheus path when SPNEGO is required
+// triggers a SPNEGO challenge, not a Bearer challenge.
+TEST_P(PrometheusTokenWebserverTest, TestMissingAuthTriggersSpnegoChallenge) {
+ curl_.set_return_headers(true);
+ Status s = curl_.FetchURL(PrometheusUrl(), &buf_);
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "401");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "WWW-Authenticate: Negotiate");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "Must authenticate with SPNEGO.");
+}
+
+// A non-Bearer Authorization header on a Prometheus path falls through to
+// SPNEGO instead of returning a Bearer challenge. An invalid SPNEGO token
+// causes SPNEGO to reject the request; the exact status code and body depend
+// on the failure mode within SPNEGO (e.g. bad base64 -> 500).
+TEST_P(PrometheusTokenWebserverTest, TestInvalidSpnegoTokenDenied) {
+ curl_.set_return_headers(true);
+ Status s = curl_.FetchURL(PrometheusUrl(), &buf_,
+ {"Authorization: Negotiate sometoken"});
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_NOT_CONTAINS(buf_.ToString(), "WWW-Authenticate: Bearer");
+}
+
+// Invalid bearer token on a Prometheus path returns 401 with a Bearer challenge.
+TEST_P(PrometheusTokenWebserverTest, TestInvalidTokenDenied) {
+ curl_.set_return_headers(true);
+ Status s = curl_.FetchURL(PrometheusUrl(), &buf_,
+ {"Authorization: Bearer wrong-token"});
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "401");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "WWW-Authenticate: Bearer");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "invalid bearer token");
+}
+
+// A token that is a valid prefix of the configured token must be rejected.
+// Guards against CRYPTO_memcmp being called with too few bytes.
+TEST_P(PrometheusTokenWebserverTest, TestPrefixTokenDenied) {
+ curl_.set_return_headers(true);
+ string prefix_token = string(kPrometheusToken).substr(0, strlen(kPrometheusToken) - 1);
+ Status s = curl_.FetchURL(PrometheusUrl(), &buf_,
+ {"Authorization: Bearer " + prefix_token});
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "401");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "WWW-Authenticate: Bearer");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "invalid bearer token");
+}
+
+// Valid bearer token on a Prometheus path succeeds without SPNEGO.
+TEST_P(PrometheusTokenWebserverTest, TestValidTokenAllowsAccess) {
+ string auth_header = Substitute("Authorization: Bearer $0", kPrometheusToken);
+ ASSERT_OK(curl_.FetchURL(PrometheusUrl(), &buf_, {auth_header}));
+ ASSERT_STR_CONTAINS(buf_.ToString(), "prometheus_metric");
+}
+
+// A fully-authenticated Kerberos principal can access a Prometheus path via
+// SPNEGO, even when a bearer token is also configured.
+TEST_P(PrometheusTokenWebserverTest, TestSpnegoAllowedOnPrometheusPath) {
+ ASSERT_OK(kdc_->Kinit("alice"));
+ curl_.set_auth(CurlAuthType::SPNEGO);
+ ASSERT_OK(curl_.FetchURL(PrometheusUrl(), &buf_));
+ ASSERT_STR_CONTAINS(buf_.ToString(), "prometheus_metric");
+}
+
+// OPTIONS requests on Prometheus paths must not require authentication.
+TEST_P(PrometheusTokenWebserverTest, TestOptionsBypassesAuthOnPrometheusPath) {
+ curl_.set_custom_method("OPTIONS");
+ curl_.set_return_headers(true);
+ // No credentials: OPTIONS must pass through.
+ ASSERT_OK(curl_.FetchURL(PrometheusUrl(), &buf_));
+ // Invalid bearer token: must also pass through (not hit reject_bearer).
+ ASSERT_OK(curl_.FetchURL(PrometheusUrl(), &buf_,
+ {"Authorization: Bearer wrong-token"}));
+}
+
+// The bearer token only bypasses SPNEGO on Prometheus paths; other paths
+// still require SPNEGO. Even a valid Prometheus bearer token is ignored on
+// non-Prometheus paths and the request is rejected.
+// TODO(KUDU-3777): once that bug is fixed the response code will be 401;
+// for now a Bearer header on a non-Prometheus path falls into the SPNEGO
+// error path which returns 500 for a non-Negotiate Authorization scheme.
+TEST_P(PrometheusTokenWebserverTest, TestTokenDoesNotBypassSpnegoOnOtherPaths) {
+ string auth_header = Substitute("Authorization: Bearer $0", kPrometheusToken);
+ Status s = curl_.FetchURL(url_, &buf_, {auth_header});
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "500");
+ ASSERT_STR_CONTAINS(buf_.ToString(), "bad Negotiate header");
+}
+
+// Invalid SPNEGO credentials on a non-Prometheus path must be rejected even
+// though a valid Prometheus bearer token is configured on the server.
+TEST_P(PrometheusTokenWebserverTest, TestInvalidSpnegoDeniedOnOtherPaths) {
+ curl_.set_return_headers(true);
+ Status s = curl_.FetchURL(url_, &buf_, {"Authorization: Negotiate aaa"});
+ ASSERT_TRUE(s.IsRemoteError()) << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "401");
+ // For an invalid SPNEGO token the server does not re-issue a
+ // WWW-Authenticate: Negotiate challenge (that only happens when no
+ // Authorization header is sent at all)
+ ASSERT_STR_NOT_CONTAINS(buf_.ToString(), "WWW-Authenticate: Negotiate");
+ // The response must not expose a Bearer challenge: the server should not
+ // hint that a bearer token would work on this path.
+ ASSERT_STR_NOT_CONTAINS(buf_.ToString(), "WWW-Authenticate: Bearer");
+}
+
// This is used to run all parameterized tests with different IP modes.
INSTANTIATE_TEST_SUITE_P(Parameters, WebserverTest,
testing::Values(IPMode::IPV4, IPMode::IPV6, IPMode::DUAL));
diff --git a/src/kudu/server/webserver.cc b/src/kudu/server/webserver.cc
index 90824ef7f..0e48121c1 100644
--- a/src/kudu/server/webserver.cc
+++ b/src/kudu/server/webserver.cc
@@ -19,6 +19,7 @@
#include <gssapi/gssapi_krb5.h>
#include <netinet/in.h>
+#include <openssl/crypto.h>
#include <sys/socket.h>
#include <algorithm>
@@ -46,6 +47,7 @@
#include "kudu/gutil/endian.h"
#include "kudu/gutil/macros.h"
#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/port.h"
#include "kudu/gutil/stl_util.h"
#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/numbers.h"
@@ -143,6 +145,31 @@ namespace kudu {
namespace {
+// Constant-time equality check for two byte sequences.
+// Returns false immediately when lengths differ (length is not considered
+// secret). For equal-length inputs, uses CRYPTO_memcmp when available
+// (OpenSSL >= 1.0.1d) and falls back to a manual XOR-accumulate loop on
+// older builds.
+bool ConstantTimeEquals(const char* a, size_t a_len, const char* b, size_t b_len) {
+ if (a_len != b_len) {
+ return false;
+ }
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL // CRYPTO_memcmp added in OpenSSL 1.0.1d
+ return CRYPTO_memcmp(a, b, a_len) == 0;
+#else
+ // XOR-accumulate loop: reads every byte regardless of content, keeping the
+ // comparison constant-time. Regular memcmp() cannot be used because its
+ // execution time depends on the data, making it vulnerable to timing
+ // side-channel attacks (see man7.org/linux/man-pages/man3/memcmp.3.html,
+ // CAVEATS).
+ unsigned char diff = 0;
+ for (size_t i = 0; i < a_len; i++) {
+ diff |= static_cast<unsigned char>(a[i]) ^ static_cast<unsigned char>(b[i]);
+ }
+ return diff == 0;
+#endif
+}
+
// Last error message from the webserver.
// TODO(todd) global strings are somewhat messy and lint is complaining
// about this. Clean this up.
@@ -249,6 +276,13 @@ Status RunSpnegoStep(const char* authz_header,
return is_complete ? Status::OK() : kIncomplete;
}
+// HTTP header name for the Authorization field.
+static constexpr const char kAuthorizationHeader[] = "Authorization";
+
+// Prefix for the Bearer authentication scheme, including the trailing space.
+static constexpr const char kBearerPrefix[] = "Bearer ";
+static constexpr size_t kBearerPrefixLen = sizeof(kBearerPrefix) - 1;
+
} // anonymous namespace
Webserver::Webserver(const WebserverOptions& opts)
@@ -399,6 +433,13 @@ Status Webserver::Start() {
#pragma GCC diagnostic pop
}
+ // Execute the token command to obtain the Prometheus bearer token.
+ if (!opts_.prometheus_token_cmd.empty()) {
+ RETURN_NOT_OK_PREPEND(
+ security::GetPasswordFromShellCommand(opts_.prometheus_token_cmd, &prometheus_token_),
+ "failed to obtain Prometheus bearer token from command");
+ }
+
options.emplace_back("listening_ports");
string listening_str;
RETURN_NOT_OK(BuildListenSpec(&listening_str));
@@ -598,52 +639,103 @@ sq_callback_result_t Webserver::BeginRequestCallback(
// The last SPNEGO step in a successful authentication may include a response
// header (e.g. when using mutual authentication).
PrerenderedWebResponse resp;
- if (opts_.require_spnego && !is_options) {
- const char* authz_header = sq_get_header(connection, "Authorization");
- string authn_princ;
- Status s = RunSpnegoStep(authz_header, &resp.response_headers, &authn_princ);
- if (s.IsIncomplete()) {
- resp.output << "Must authenticate with SPNEGO.";
- resp.status_code = HttpStatusCode::AuthenticationRequired;
- SendResponse(connection, &resp);
- return SQ_HANDLED_OK;
- }
- if (s.ok() && authn_princ.empty()) {
- s = Status::RuntimeError("SPNEGO indicated complete, but got empty principal");
- // Crash in debug builds, but fall through to treating as an error 500 in
- // release.
- LOG(DFATAL) << "Got no authenticated principal for SPNEGO-authenticated "
- << " connection from "
- << GetRemoteAddress(request_info).ToString()
- << ": " << s.ToString();
- }
- if (!s.ok()) {
- LOG(WARNING) << "Failed to authenticate request from "
- << GetRemoteAddress(request_info).ToString()
- << " via SPNEGO: " << s.ToString();
- resp.output << s.ToString();
- resp.status_code = s.IsNotAuthorized() ?
- HttpStatusCode::AuthenticationRequired :
- HttpStatusCode::InternalServerError;
- SendResponse(connection, &resp);
- return SQ_HANDLED_OK;
- }
- if (opts_.spnego_post_authn_callback) {
- opts_.spnego_post_authn_callback(authn_princ);
+ // Prometheus endpoints support two authentication mechanisms:
+ // 1. Bearer token ('Authorization: Bearer <token>'): intended for Prometheus
+ // scrapers, which do not support SPNEGO. Enabled by setting
+ // --webserver_prometheus_token_cmd.
+ // 2. SPNEGO: used by human operators or proxies. Falls through to the
+ // standard SPNEGO block below if SPNEGO is required.
+ // When a Prometheus path receives a Bearer-scheme Authorization header and a
+ // token is configured, the token is validated here and SPNEGO is skipped.
+ // All other requests (including Prometheus paths without a Bearer header)
+ // proceed to the SPNEGO block if SPNEGO is required.
+ bool is_prometheus_path = false;
+ {
+ shared_lock l(lock_);
+ is_prometheus_path = ContainsKey(prometheus_paths_, request_info->uri);
+ }
+
+ auto reject_bearer = [&](const char* reason) -> sq_callback_result_t {
+ LOG(WARNING) << "Failed to authenticate Prometheus request from "
+ << GetRemoteAddress(request_info).ToString()
+ << ": " << reason;
+ resp.response_headers.emplace("WWW-Authenticate", "Bearer");
+ resp.output << "Must authenticate with a valid bearer token: " << reason;
+ resp.status_code = HttpStatusCode::AuthenticationRequired;
+ SendResponse(connection, &resp);
+ return SQ_HANDLED_OK;
+ };
+
+ bool authenticated_via_prometheus_token = false;
+ if (is_prometheus_path && !prometheus_token_.empty() && !is_options) {
+ const char* authz_header = sq_get_header(connection, kAuthorizationHeader);
+ if (authz_header != nullptr &&
+ strncmp(authz_header, kBearerPrefix, kBearerPrefixLen) == 0) {
+ const char* provided_token = authz_header + kBearerPrefixLen;
+ if (!ConstantTimeEquals(provided_token, strlen(provided_token),
+ prometheus_token_.c_str(),
+ prometheus_token_.size())) {
+ return reject_bearer("invalid bearer token");
+ }
+ // Token is valid; skip SPNEGO and proceed directly to the handler.
+ authenticated_via_prometheus_token = true;
}
+ }
- string local_user;
- s = security::MapPrincipalToLocalName(authn_princ, &local_user);
- if (!s.ok()) {
- LOG(WARNING) << "Failed to map Kerberos principal '" << authn_princ
- << "' to local name: " << s.ToString();
- resp.output << s.ToString();
- resp.status_code = HttpStatusCode::InternalServerError;
- SendResponse(connection, &resp);
- return SQ_HANDLED_OK;
+ if (!authenticated_via_prometheus_token) {
+ if (opts_.require_spnego && !is_options) {
+ const char* authz_header = sq_get_header(connection, kAuthorizationHeader);
+ string authn_princ;
+ Status s = RunSpnegoStep(authz_header, &resp.response_headers, &authn_princ);
+ if (s.IsIncomplete()) {
+ resp.output << "Must authenticate with SPNEGO.";
+ resp.status_code = HttpStatusCode::AuthenticationRequired;
+ SendResponse(connection, &resp);
+ return SQ_HANDLED_OK;
+ }
+ if (s.ok() && authn_princ.empty()) {
+ s = Status::RuntimeError("SPNEGO indicated complete, but got empty principal");
+ // Crash in debug builds, but fall through to treating as an error 500 in
+ // release.
+ LOG(DFATAL) << "Got no authenticated principal for SPNEGO-authenticated "
+ << " connection from "
+ << GetRemoteAddress(request_info).ToString()
+ << ": " << s.ToString();
+ }
+ if (!s.ok()) {
+ LOG(WARNING) << "Failed to authenticate request from "
+ << GetRemoteAddress(request_info).ToString()
+ << " via SPNEGO: " << s.ToString();
+ resp.output << s.ToString();
+ resp.status_code = s.IsNotAuthorized() ?
+ HttpStatusCode::AuthenticationRequired :
+ HttpStatusCode::InternalServerError;
+ SendResponse(connection, &resp);
+ return SQ_HANDLED_OK;
+ }
+
+ if (opts_.spnego_post_authn_callback) {
+ opts_.spnego_post_authn_callback(authn_princ);
+ }
+
+ string local_user;
+ s = security::MapPrincipalToLocalName(authn_princ, &local_user);
+ if (!s.ok()) {
+ LOG(WARNING) << "Failed to map Kerberos principal '" << authn_princ
+ << "' to local name: " << s.ToString();
+ resp.output << s.ToString();
+ resp.status_code = HttpStatusCode::InternalServerError;
+ SendResponse(connection, &resp);
+ return SQ_HANDLED_OK;
+ }
+ request_info->remote_user = strdup(local_user.c_str());
+ } else if (is_prometheus_path && !prometheus_token_.empty() && !is_options) {
+ // SPNEGO is not required, but a bearer token is configured and the client
+ // did not supply one. Reject with a Bearer challenge. OPTIONS requests
+ // are exempt.
+ return reject_bearer("missing or non-Bearer authorization header");
}
- request_info->remote_user = strdup(local_user.c_str());
}
PathHandler* handler = nullptr;
@@ -1040,6 +1132,11 @@ void Webserver::RegisterJsonPathHandler(
callback));
}
+void Webserver::MarkPathAsPrometheus(const string& path) {
+ std::lock_guard l(lock_);
+ prometheus_paths_.insert(path);
+}
+
bool Webserver::MustacheTemplateAvailable(const string& path) const {
if (!static_pages_available()) {
return false;
diff --git a/src/kudu/server/webserver.h b/src/kudu/server/webserver.h
index 4720cd7e2..718daaad8 100644
--- a/src/kudu/server/webserver.h
+++ b/src/kudu/server/webserver.h
@@ -22,12 +22,12 @@
#include <map>
#include <string>
#include <unordered_map>
+#include <unordered_set>
#include <utility>
#include <vector>
#include <squeasel.h>
-#include "kudu/gutil/port.h"
#include "kudu/server/webserver_options.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/rw_mutex.h"
@@ -100,6 +100,12 @@ class Webserver : public WebCallbackRegistry {
// True if serving all traffic over SSL, false otherwise
bool IsSecure() const;
+ // Marks 'path' as a Prometheus endpoint. When --webserver_prometheus_token
+ // is set, requests to these paths may authenticate via
+ // 'Authorization: Bearer <token>' instead of SPNEGO, allowing Prometheus
+ // (which does not support SPNEGO) to scrape a secured cluster.
+ void MarkPathAsPrometheus(const std::string& path);
+
// Change the status to true once the webserver's startup is completed. The startup
// of a kudu server is split into two parts initialization and starting phase. In the
// initialization phase we start the webserver, register only the path handlers which are ready
@@ -212,6 +218,11 @@ class Webserver : public WebCallbackRegistry {
const WebserverOptions opts_;
+ // The effective bearer token for Prometheus endpoint authentication.
+ // Populated in Start() by running opts_.prometheus_token_cmd.
+ // Empty if bearer token auth is not configured.
+ std::string prometheus_token_;
+
// Lock guarding the path_handlers_ map and footer_html.
RWMutex lock_;
@@ -235,6 +246,10 @@ class Webserver : public WebCallbackRegistry {
// Handle to Mongoose context; owned and freed by Mongoose internally
struct sq_context* context_;
+ // Set of paths that accept bearer token authentication as an alternative to
+ // SPNEGO. Protected by 'lock_'.
+ std::unordered_set<std::string> prometheus_paths_;
+
// Hold the status of the webserver's startup status.
std::atomic<bool> is_started_;
diff --git a/src/kudu/server/webserver_options.cc b/src/kudu/server/webserver_options.cc
index 826735f7a..51b0634c2 100644
--- a/src/kudu/server/webserver_options.cc
+++ b/src/kudu/server/webserver_options.cc
@@ -100,6 +100,18 @@ DEFINE_string(webserver_password_file, "",
"--webserver_require_spnego flag)");
TAG_FLAG(webserver_password_file, sensitive);
+DEFINE_string(webserver_prometheus_token_cmd, "",
+ "A Unix command whose output is used as the bearer token for Prometheus "
+ "metrics endpoint authentication. When set, requests to the Prometheus "
+ "endpoints (/metrics_prometheus, /prometheus-sd) may authenticate via "
+ "'Authorization: Bearer <token>' as an alternative to SPNEGO. This is "
+ "primarily useful together with --webserver_require_spnego: since "
+ "Prometheus does not support SPNEGO, the bearer token allows it to scrape "
+ "a secured Kudu cluster. Clients that do support SPNEGO can still use it "
+ "on these endpoints. Requires TLS to be configured for the embedded "
+ "webserver (--webserver_certificate_file).");
+TAG_FLAG(webserver_prometheus_token_cmd, sensitive);
+
DEFINE_int32(webserver_num_worker_threads, 50,
"Maximum number of threads to start for handling webserver requests");
TAG_FLAG(webserver_num_worker_threads, advanced);
@@ -187,6 +199,19 @@ bool ValidateTlsMinVersion(const char* /* flagname */, const string& ver) {
}
DEFINE_validator(webserver_tls_min_protocol, &ValidateTlsMinVersion);
+bool ValidatePrometheusTokenFlags() {
+ const bool has_token_cmd = !FLAGS_webserver_prometheus_token_cmd.empty();
+ const bool has_tls = !FLAGS_webserver_certificate_file.empty();
+
+ if (has_token_cmd && !has_tls) {
+ LOG(ERROR) << "--webserver_prometheus_token_cmd requires TLS to be configured "
+ "for the embedded webserver (--webserver_certificate_file).";
+ return false;
+ }
+ return true;
+}
+GROUP_FLAG_VALIDATOR(webserver_prometheus_token_options, ValidatePrometheusTokenFlags);
+
}; // anonymous namespace
namespace kudu {
@@ -209,6 +234,7 @@ WebserverOptions::WebserverOptions()
private_key_password_cmd(FLAGS_webserver_private_key_password_cmd),
authentication_domain(FLAGS_webserver_authentication_domain),
password_file(FLAGS_webserver_password_file),
+ prometheus_token_cmd(FLAGS_webserver_prometheus_token_cmd),
tls_ciphers(FLAGS_webserver_tls_ciphers),
tls_ciphersuites(FLAGS_webserver_tls_ciphersuites),
tls_min_protocol(FLAGS_webserver_tls_min_protocol),
diff --git a/src/kudu/server/webserver_options.h b/src/kudu/server/webserver_options.h
index 2df2243b5..91ab8bbc3 100644
--- a/src/kudu/server/webserver_options.h
+++ b/src/kudu/server/webserver_options.h
@@ -39,6 +39,7 @@ struct WebserverOptions {
std::string private_key_password_cmd;
std::string authentication_domain;
std::string password_file;
+ std::string prometheus_token_cmd;
std::string tls_ciphers;
std::string tls_ciphersuites;
std::string tls_min_protocol;
diff --git a/src/kudu/util/mini_prometheus.cc b/src/kudu/util/mini_prometheus.cc
index 690374e4b..a22edb7f0 100644
--- a/src/kudu/util/mini_prometheus.cc
+++ b/src/kudu/util/mini_prometheus.cc
@@ -131,6 +131,21 @@ Status MiniPrometheus::WriteConfig() const {
out << YAML::Key << "metrics_path" << YAML::Value << options_.metrics_path;
out << YAML::Key << "scrape_interval" << YAML::Value << options_.scrape_interval;
+ if (!options_.bearer_token.empty()) {
+ out << YAML::Key << "authorization" << YAML::Value << YAML::BeginMap;
+ out << YAML::Key << "type" << YAML::Value << "Bearer";
+ out << YAML::Key << "credentials" << YAML::Value << options_.bearer_token;
+ out << YAML::EndMap;
+ }
+ if (options_.scheme != "http") {
+ out << YAML::Key << "scheme" << YAML::Value << options_.scheme;
+ }
+ if (options_.skip_tls_verify) {
+ out << YAML::Key << "tls_config" << YAML::Value << YAML::BeginMap;
+ out << YAML::Key << "insecure_skip_verify" << YAML::Value << true;
+ out << YAML::EndMap;
+ }
+
if (!options_.static_targets.empty()) {
out << YAML::Key << "static_configs" << YAML::Value << YAML::BeginSeq;
out << YAML::BeginMap;
@@ -149,6 +164,17 @@ Status MiniPrometheus::WriteConfig() const {
out << YAML::BeginMap;
out << YAML::Key << "url" << YAML::Value << url;
out << YAML::Key << "refresh_interval" << YAML::Value << options_.sd_refresh_interval;
+ if (options_.skip_tls_verify) {
+ out << YAML::Key << "tls_config" << YAML::Value << YAML::BeginMap;
+ out << YAML::Key << "insecure_skip_verify" << YAML::Value << true;
+ out << YAML::EndMap;
+ }
+ if (!options_.bearer_token.empty()) {
+ out << YAML::Key << "authorization" << YAML::Value << YAML::BeginMap;
+ out << YAML::Key << "type" << YAML::Value << "Bearer";
+ out << YAML::Key << "credentials" << YAML::Value << options_.bearer_token;
+ out << YAML::EndMap;
+ }
out << YAML::EndMap;
}
out << YAML::EndSeq;
@@ -182,6 +208,11 @@ Status MiniPrometheus::GetTargets(rapidjson::Document* doc) {
}
Status MiniPrometheus::WaitForActiveTargets(int count, MonoDelta timeout) {
+ return WaitForTargetHealth(count, "up", timeout);
+}
+
+Status MiniPrometheus::WaitForTargetHealth(int count, const string& health,
+ MonoDelta timeout) {
const MonoTime deadline = MonoTime::Now() + timeout;
while (MonoTime::Now() < deadline) {
rapidjson::Document doc;
@@ -189,21 +220,22 @@ Status MiniPrometheus::WaitForActiveTargets(int count, MonoDelta timeout) {
if (s.ok() && doc.IsObject() && doc.HasMember("data") &&
doc["data"].HasMember("activeTargets")) {
const auto& active = doc["data"]["activeTargets"];
- int healthy = 0;
+ int matched = 0;
for (rapidjson::SizeType i = 0; i < active.Size(); ++i) {
if (active[i].HasMember("health") &&
- string(active[i]["health"].GetString()) == "up") {
- ++healthy;
+ string(active[i]["health"].GetString()) == health) {
+ ++matched;
}
}
- if (healthy >= count) {
+ if (matched >= count) {
return Status::OK();
}
}
SleepFor(MonoDelta::FromSeconds(1));
}
return Status::TimedOut(
- Substitute("Timed out waiting for $0 active Prometheus targets", count));
+ Substitute("Timed out waiting for $0 Prometheus targets with health='$1'",
+ count, health));
}
Status MiniPrometheus::Query(const string& promql, rapidjson::Document* doc) {
diff --git a/src/kudu/util/mini_prometheus.h b/src/kudu/util/mini_prometheus.h
index 1dca5130e..4fee5c6b1 100644
--- a/src/kudu/util/mini_prometheus.h
+++ b/src/kudu/util/mini_prometheus.h
@@ -45,6 +45,21 @@ struct MiniPrometheusOptions {
// HTTP path used to scrape metrics on discovered targets.
std::string metrics_path = "/metrics_prometheus";
+ // If non-empty, emitted as an 'authorization: Bearer' section in both the
+ // scrape_configs block (for metric scraping) and each http_sd_configs entry
+ // (for SD discovery). A single token is used for both because Kudu applies
+ // the same --webserver_prometheus_token_cmd to all Prometheus endpoints
+ // (/metrics_prometheus and /prometheus-sd) uniformly.
+ std::string bearer_token;
+
+ // URL scheme used for scraping targets. Defaults to "http". Set to "https"
+ // when the Kudu webservers are TLS-enabled.
+ std::string scheme = "http";
+
+ // When true, Prometheus skips TLS certificate verification for both scrape
+ // and SD requests.
+ bool skip_tls_verify = false;
+
// How frequently Prometheus scrapes discovered targets.
std::string scrape_interval = "2s";
@@ -72,6 +87,10 @@ class MiniPrometheus {
// polling every second.
Status WaitForActiveTargets(int count, MonoDelta timeout);
+ // Blocks until at least 'count' targets report the given 'health' state
+ // ("up" or "down"), polling every second.
+ Status WaitForTargetHealth(int count, const std::string& health, MonoDelta timeout);
+
// Issues a PromQL instant query via /api/v1/query and populates 'doc' with
// the parsed JSON response.
Status Query(const std::string& promql,