You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/19 09:44:13 UTC
(tooling-llmao) branch main updated: sampler: the one reader of /metrics, KV cache size included
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
The following commit(s) were added to refs/heads/main by this push:
new d13342e sampler: the one reader of /metrics, KV cache size included
d13342e is described below
commit d13342eaa1de9178d7c4d8d19b7295c37ef0fdda
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Mon Oct 19 04:43:30 2026 -0500
sampler: the one reader of /metrics, KV cache size included
The fleet read each server's /metrics once, on the edge into Healthy,
for its KV cache size; the sampler reads it every interval. Two readers
of one endpoint, and the fleet's could miss: a failed scrape was only
retried on the next probe that found the size still unknown.
The sampler now carries the KV cache size on each reading and sets it
on the server, so a restarted vLLM that sized its cache differently is
seen within an interval. parse_kv_cache_tokens moves to sampler.py
unchanged. The fleet keeps the one read that needs the server's key,
the served window from /v1/models (fetch_max_model_len), and no longer
asks for /metrics.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
docs/STATUS.md | 2 +-
llmao/capacity.py | 3 ++
llmao/fleet.py | 107 +++++++++------------------------------------
llmao/sampler.py | 75 +++++++++++++++++++++++++++----
tests/test_fleet_health.py | 89 ++++++-------------------------------
tests/test_sampler.py | 75 +++++++++++++++++++++++++++++--
6 files changed, 178 insertions(+), 173 deletions(-)
diff --git a/docs/STATUS.md b/docs/STATUS.md
index 72566a1..10991cf 100644
--- a/docs/STATUS.md
+++ b/docs/STATUS.md
@@ -55,7 +55,7 @@ its usage prices at zero.
Limits of that sizing, all in `hosting/README.md`: it happens once, at install; re-running the install on a box whose servers are already up sees little free memory and sets no fraction; and servers that load at the same moment get less KV cache than their share.
-**Observed state:** the `/models` drill-down shows measured KV cache against the served context window, scraped from vLLM's `/metrics` on the transition into Healthy. A `--max-model-len` above what the cache holds makes vLLM hang rather than error, which was previously only visible by reading a startup log.
+**Observed state:** the `/models` drill-down shows measured KV cache against the served context window, the cache size taken from each reading of vLLM's `/metrics` by the sampler, and the window from `/v1/models` on the transition into Healthy. A `--max-model-len` above what the cache holds makes vLLM hang rather than error, which was previously only visible by reading a startup log.
Remaining: long vLLM boot, config revision on the box, pending-assignment table, box smoke, and nothing restarting a GPU box automatically.
diff --git a/llmao/capacity.py b/llmao/capacity.py
index 386a069..64e2c12 100644
--- a/llmao/capacity.py
+++ b/llmao/capacity.py
@@ -80,6 +80,9 @@ class Sample:
counters: dict[str, float] = field(default_factory=dict)
# Queue-time histogram: upper bound in seconds => cumulative count.
buckets: dict[float, float] = field(default_factory=dict)
+ # The KV cache size in tokens, fixed at engine start. Not rolled up: the
+ # sampler hands it to the fleet for the oversized check.
+ kv_cache_tokens: int | None = None
def retention_days(cfg: Any) -> int:
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 99ede8e..a53101d 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -340,8 +340,9 @@ class VllmServer:
# was ordered.
self.vram_gb = vram_gb
self.disk_gb = disk_gb
- # Observed from the box, not configured. None until the server has
- # been SERVING and a scrape has succeeded.
+ # Observed from the box, not configured. None until read: the KV
+ # cache size by the sampler from /metrics, the served window (and
+ # observed_at) by the fleet from /v1/models once SERVING.
#
# kv_cache_tokens is the figure that decides whether max_model_len is
# safe: a serving length above the measured cache produces HANGS, not
@@ -1011,18 +1012,16 @@ class Fleet:
reached=reached,
in_litellm=bool(in_litellm_by_srv.get(srv, False)),
)
- # Scrape on the edge into SERVING, and once more if an earlier
- # attempt came back empty. Both values are fixed at engine
- # init, so re-reading them every probe would be waste.
- if srv.state == HEALTHY and (was != HEALTHY or srv.kv_cache_tokens is None):
+ # Read the served window on the edge into SERVING, and again
+ # if an earlier attempt came back empty. It is fixed at engine
+ # init, so re-reading it every probe would be waste. The KV
+ # cache size comes from the sampler.
+ if srv.state == HEALTHY and (was != HEALTHY or srv.observed_max_model_len is None):
root = srv.root_url
if root:
- kv, mml = await fetch_observed(client, root, srv.api_key)
- if kv is not None:
- srv.kv_cache_tokens = kv
+ mml = await fetch_max_model_len(client, root, srv.api_key)
if mml is not None:
srv.observed_max_model_len = mml
- if kv is not None or mml is not None:
srv.observed_at = stamp
finally:
if own:
@@ -1235,60 +1234,6 @@ def remove_refusal(dep) -> str | None:
return None
-def parse_kv_cache_tokens(metrics_text: str) -> int | None:
- """KV cache size in tokens from vLLM's Prometheus output, or None.
-
- vLLM reports cache geometry as labels on an Info-style metric:
-
- vllm:cache_config_info{block_size="16",num_gpu_blocks="33960",
- kv_cache_size_tokens="855836",...} 1.0
-
- Prefer kv_cache_size_tokens, which is the same number vLLM prints at
- startup as "GPU KV cache size: N tokens".
-
- Do NOT compute it as num_gpu_blocks * block_size. On the A100 serving
- Gemma those give 543,360 against a reported 855,836 -- blocks do not map
- uniformly to tokens for a model with heterogeneous head dimensions, and
- the product understates by a third. Understating the cache would falsely
- flag a correctly-sized server as oversized.
-
- The product is kept only as a fallback for builds that do not emit the
- token count, where an approximate figure beats none.
-
- Returns None rather than raising when the metric is absent or shaped
- differently -- vLLM's metric names are not a stable API, and a missing
- figure should degrade the display rather than break the probe loop.
- """
- for line in metrics_text.splitlines():
- if not line.startswith("vllm:cache_config_info"):
- continue
- start, end = line.find("{"), line.rfind("}")
- if start < 0 or end < start:
- continue
- labels: dict[str, str] = {}
- for part in line[start + 1 : end].split(","):
- k, _, v = part.partition("=")
- labels[k.strip()] = v.strip().strip('"')
-
- direct = labels.get("kv_cache_size_tokens")
- if direct and direct != "None":
- try:
- tokens = int(direct)
- except ValueError:
- tokens = 0
- if tokens > 0:
- return tokens
-
- try:
- blocks = int(labels["num_gpu_blocks"])
- size = int(labels["block_size"])
- except (KeyError, ValueError):
- continue
- if blocks > 0 and size > 0:
- return blocks * size
- return None
-
-
def parse_max_model_len(models_json: Any) -> int | None:
"""Served context window from /v1/models, or None."""
try:
@@ -1305,38 +1250,28 @@ def parse_max_model_len(models_json: Any) -> int | None:
return None
-async def fetch_observed(client: httpx.AsyncClient, base: str, api_key: str) -> tuple[int | None, int | None]:
- """Scrape (kv_cache_tokens, max_model_len) from a serving vLLM.
-
- base is the server root, not api_base: /metrics sits at the root and the
- model list at /v1/models, so a /v1-suffixed base would 404 the model list
- and leave both fields unknown.
+async def fetch_max_model_len(client: httpx.AsyncClient, base: str, api_key: str) -> int | None:
+ """The served context window, from a serving vLLM's /v1/models, or None.
- Both are fixed at engine init, so this is called on the transition into
- SERVING rather than on every probe -- polling an unchanging value every
- 45s would be waste.
+ base is the server root, not api_base: a /v1-suffixed base would ask for
+ /v1/v1/models and 404. The window is fixed at engine init, so this is
+ called on the transition into SERVING, not every probe. /v1/models needs
+ the server's key.
- /metrics is unauthenticated on vLLM; /v1/models is not, so the key goes on
- both rather than reasoning about which needs it.
+ The KV cache size, the other half of the oversized check, is read by the
+ sampler (llmao/sampler.py), the one reader of /metrics.
- Never raises: an unreachable or differently-shaped endpoint leaves the
- fields None and the UX shows them as unknown.
+ Never raises: an unreachable or differently-shaped endpoint leaves it
+ unknown and the UX shows it as such.
"""
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
- kv = mml = None
- try:
- resp = await client.get(f"{base}/metrics", headers=headers)
- if resp.status_code == 200:
- kv = parse_kv_cache_tokens(resp.text)
- except httpx.HTTPError:
- pass
try:
resp = await client.get(f"{base}/v1/models", headers=headers)
if resp.status_code == 200:
- mml = parse_max_model_len(resp.json())
+ return parse_max_model_len(resp.json())
except (httpx.HTTPError, ValueError):
pass
- return kv, mml
+ return None
async def _get_health(client: httpx.AsyncClient, url: str) -> tuple[bool, bool, str | None]:
diff --git a/llmao/sampler.py b/llmao/sampler.py
index 51490f7..c49fe1d 100644
--- a/llmao/sampler.py
+++ b/llmao/sampler.py
@@ -92,6 +92,60 @@ def _labels(inner: str) -> dict[str, str]:
return labels
+def parse_kv_cache_tokens(metrics_text: str) -> int | None:
+ """KV cache size in tokens from vLLM's Prometheus output, or None.
+
+ vLLM reports cache geometry as labels on an Info-style metric:
+
+ vllm:cache_config_info{block_size="16",num_gpu_blocks="33960",
+ kv_cache_size_tokens="855836",...} 1.0
+
+ Prefer kv_cache_size_tokens, which is the same number vLLM prints at
+ startup as "GPU KV cache size: N tokens".
+
+ Do NOT compute it as num_gpu_blocks * block_size. On the A100 serving
+ Gemma those give 543,360 against a reported 855,836 -- blocks do not map
+ uniformly to tokens for a model with heterogeneous head dimensions, and
+ the product understates by a third. Understating the cache would falsely
+ flag a correctly-sized server as oversized.
+
+ The product is kept only as a fallback for builds that do not emit the
+ token count, where an approximate figure beats none.
+
+ Returns None rather than raising when the metric is absent or shaped
+ differently -- vLLM's metric names are not a stable API, and a missing
+ figure should degrade the display rather than break the probe loop.
+ """
+ for line in metrics_text.splitlines():
+ if not line.startswith("vllm:cache_config_info"):
+ continue
+ start, end = line.find("{"), line.rfind("}")
+ if start < 0 or end < start:
+ continue
+ labels: dict[str, str] = {}
+ for part in line[start + 1 : end].split(","):
+ k, _, v = part.partition("=")
+ labels[k.strip()] = v.strip().strip('"')
+
+ direct = labels.get("kv_cache_size_tokens")
+ if direct and direct != "None":
+ try:
+ tokens = int(direct)
+ except ValueError:
+ tokens = 0
+ if tokens > 0:
+ return tokens
+
+ try:
+ blocks = int(labels["num_gpu_blocks"])
+ size = int(labels["block_size"])
+ except (KeyError, ValueError):
+ continue
+ if blocks > 0 and size > 0:
+ return blocks * size
+ return None
+
+
def parse_metrics(text: str, *, now: float | None = None) -> Sample:
"""What the rollup needs, from vLLM's Prometheus exposition text.
@@ -99,7 +153,7 @@ def parse_metrics(text: str, *, now: float | None = None) -> Sample:
format. Anything unreadable is skipped, so a box answering with HTML or
nothing gives an empty reading rather than an error.
"""
- out = Sample(ts=now if now is not None else time.time())
+ out = Sample(ts=now if now is not None else time.time(), kv_cache_tokens=parse_kv_cache_tokens(text))
for line in text.splitlines():
if not line or line.startswith("#"):
continue
@@ -143,8 +197,8 @@ class MetricsSampler:
self._timeout = timeout_s
self._down: set[str] = set()
- def targets(self) -> list[tuple[str, str, str]]:
- """(server, model_name, metrics URL) for each self-hosted server with an address.
+ def targets(self) -> list[tuple[Any, str, str, str]]:
+ """(VllmServer, server, model_name, metrics URL) for each self-hosted server with an address.
A server is host:listen port, as everywhere else in llmao. The URL
is its public address, the one LiteLLM uses.
@@ -155,7 +209,7 @@ class MetricsSampler:
if not dep.self_hosted or srv is None or not dep.api_base:
continue
root = dep.api_base.rstrip("/").removesuffix("/v1")
- out.append((f"{srv.host}:{srv.listen_port}", dep.model_name, f"{root}/metrics"))
+ out.append((srv, f"{srv.host}:{srv.listen_port}", dep.model_name, f"{root}/metrics"))
return out
async def _read(self, client: httpx.AsyncClient, server: str, url: str) -> Sample | None:
@@ -176,12 +230,17 @@ class MetricsSampler:
"""One reading of every box, all at once: twenty boxes in turn could
outlast the interval."""
targets = self.targets()
- readings = await asyncio.gather(*(self._read(client, server, url) for server, _model, url in targets))
- for (server, model_name, _url), sample in zip(targets, readings, strict=True):
+ readings = await asyncio.gather(*(self._read(client, server, url) for _srv, server, _model, url in targets))
+ for (srv, server, model_name, _url), sample in zip(targets, readings, strict=True):
if sample is None:
self._rollup.missed(server)
- else:
- self._rollup.record(server, model_name, sample)
+ continue
+ self._rollup.record(server, model_name, sample)
+ # The KV cache size, for the /models drill-down and the oversized
+ # check. Fixed at engine start, but taken from every reading: a
+ # restarted vLLM may have sized it differently.
+ if sample.kv_cache_tokens is not None:
+ srv.kv_cache_tokens = sample.kv_cache_tokens
async def run(self) -> None:
async with httpx.AsyncClient() as client:
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index 3d35350..770763f 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -27,8 +27,7 @@ from llmao.fleet import (
FleetDeployment,
VllmServer,
add_refusal,
- fetch_observed,
- parse_kv_cache_tokens,
+ fetch_max_model_len,
parse_max_model_len,
)
from llmao.model_status import (
@@ -441,58 +440,6 @@ def test_never_healthy_box_goes_starting_then_down():
assert s.state == STALLED
-_METRICS = """\
-# HELP vllm:num_requests_running Number of requests in model execution batches.
-# TYPE vllm:num_requests_running gauge
-vllm:num_requests_running{model_name="google/gemma-4-26B-A4B-it"} 0.0
-# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig
-# TYPE vllm:cache_config_info gauge
-vllm:cache_config_info{block_size="16",cache_dtype="auto",enable_prefix_caching="True",gpu_memory_utilization="0.9",kv_cache_max_concurrency="6.52951355508556",kv_cache_memory_bytes="None",kv_cache_size_tokens="855836",num_cpu_blocks="None",num_gpu_blocks="33960"} 1.0
-"""
-
-
-def test_parse_kv_cache_tokens_prefers_the_reported_count():
- """kv_cache_size_tokens is authoritative; the block product is not.
-
- Real output from the A100 serving Gemma reports 855,836 tokens while
- num_gpu_blocks * block_size gives 543,360 -- blocks do not map uniformly
- to tokens for a model with heterogeneous head dimensions. Taking the
- product would understate the cache by a third and falsely flag a
- correctly-sized server as oversized.
- """
- assert parse_kv_cache_tokens(_METRICS) == 855836
- assert 33960 * 16 == 543360 # what the wrong answer would have been
-
-
-def test_parse_kv_cache_tokens_falls_back_to_block_product():
- """Builds that do not emit the token count still get an approximation."""
- text = 'vllm:cache_config_info{block_size="16",num_gpu_blocks="1000"} 1.0'
- assert parse_kv_cache_tokens(text) == 16000
-
- text = 'vllm:cache_config_info{block_size="16",num_gpu_blocks="1000",kv_cache_size_tokens="None"} 1.0'
- assert parse_kv_cache_tokens(text) == 16000
-
-
-def test_parse_kv_cache_tokens_absent():
- """vLLM's metric names are not a stable API.
-
- A missing or renamed metric degrades the display rather than breaking the
- probe loop, so this returns None instead of raising.
- """
- assert parse_kv_cache_tokens("vllm:num_requests_running{} 0.0\n") is None
- assert parse_kv_cache_tokens("") is None
-
-
-def test_parse_kv_cache_tokens_malformed():
- for text in (
- 'vllm:cache_config_info{block_size="16"} 1.0', # no block count
- 'vllm:cache_config_info{block_size="x",num_gpu_blocks="1"} 1.0',
- "vllm:cache_config_info block_size=16 1.0", # no braces
- 'vllm:cache_config_info{block_size="0",num_gpu_blocks="0"} 1.0',
- ):
- assert parse_kv_cache_tokens(text) is None, text
-
-
def test_parse_max_model_len():
body = {"data": [{"id": "gemma4-26b", "max_model_len": 131072}]}
assert parse_max_model_len(body) == 131072
@@ -539,13 +486,12 @@ def test_oversized_falls_back_to_configured_length():
assert s.oversized is True
-def test_fetch_observed_scrapes_from_the_root():
- """fetch_observed takes the server root, not a /v1-suffixed api_base.
+def test_the_served_window_is_read_from_the_root():
+ """fetch_max_model_len takes the server root, not a /v1-suffixed api_base.
- /metrics sits at the root and the model list at /v1/models. A /v1-
- suffixed base would request root/v1/metrics and root/v1/v1/models,
- 404 both, and leave the fields None -- oversized detection quietly dead
- while the rest of the suite stays green.
+ A /v1-suffixed base would ask for root/v1/v1/models, 404, and leave the
+ window unknown -- oversized detection quietly dead while the rest of the
+ suite stays green.
"""
calls = []
root = "http://10.0.0.1:8001"
@@ -553,25 +499,21 @@ def test_fetch_observed_scrapes_from_the_root():
class _Client:
async def get(self, url, headers=None):
calls.append(url)
- if url == f"{root}/metrics":
- return _Resp(200, text=_METRICS)
if url == f"{root}/v1/models":
return _Resp(200, body={"data": [{"max_model_len": 131072}]})
return _Resp(404)
- kv, mml = asyncio.run(fetch_observed(_Client(), root, "sk-x"))
- assert kv == 855836
- assert mml == 131072
- assert calls == [f"{root}/metrics", f"{root}/v1/models"]
+ assert asyncio.run(fetch_max_model_len(_Client(), root, "sk-x")) == 131072
+ assert calls == [f"{root}/v1/models"]
def test_probe_all_scrapes_from_root_not_api_base():
- """The SERVING-transition scrape must be handed root_url, not api_base.
+ """The SERVING-transition read must be handed root_url, not api_base.
- Catches the regression one layer up from fetch_observed: if probe_all
- ever passes the /v1-suffixed base, both scrape requests 404 on every
- transition, the KV column on the /models drill-down goes empty, and
- oversized stays unknown without any other test failing.
+ Catches the regression one layer up from fetch_max_model_len: if
+ probe_all ever passes the /v1-suffixed base, the read 404s on every
+ transition and oversized stays unknown without any other test failing.
+ /metrics is the sampler's, so the fleet no longer asks for it.
"""
s = _server()
calls = []
@@ -582,8 +524,6 @@ def test_probe_all_scrapes_from_root_not_api_base():
calls.append(url)
if url.endswith("/health"):
return _Resp(200, text="OK")
- if url.endswith("/metrics"):
- return _Resp(200, text=_METRICS)
if url.endswith("/v1/models"):
return _Resp(200, body={"data": [{"max_model_len": 131072}]})
return _Resp(404)
@@ -593,9 +533,8 @@ def test_probe_all_scrapes_from_root_not_api_base():
asyncio.run(fleet.probe_all(client=_Client(), now=1.0))
assert s.state == HEALTHY
- assert s.kv_cache_tokens == 855836
assert s.observed_max_model_len == 131072
- assert calls == [f"{root}/health", f"{root}/metrics", f"{root}/v1/models"]
+ assert calls == [f"{root}/health", f"{root}/v1/models"]
def test_the_fleet_is_ready_after_its_first_pass_even_a_failed_one():
diff --git a/tests/test_sampler.py b/tests/test_sampler.py
index bb5fcf2..a587281 100644
--- a/tests/test_sampler.py
+++ b/tests/test_sampler.py
@@ -28,7 +28,7 @@ from easydict import EasyDict as edict # noqa: N813
from llmao.capacity import HOUR_S, HourlyRollup
from llmao.fleet import Fleet, FleetDeployment, VllmServer
-from llmao.sampler import MetricsSampler, parse_metrics, sample_interval
+from llmao.sampler import MetricsSampler, parse_kv_cache_tokens, parse_metrics, sample_interval
# As vLLM 0.27 writes it, trimmed.
METRICS = "\n".join(
@@ -108,11 +108,12 @@ def _server(host, *, public_port=8001):
def test_each_server_is_named_by_host_and_listen_port():
- fleet = Fleet(None, [_server("203.0.113.10", public_port=22018), _server("198.51.100.8", public_port=None)])
+ reachable = _server("203.0.113.10", public_port=22018)
+ fleet = Fleet(None, [reachable, _server("198.51.100.8", public_port=None)])
fleet.deployments.append(FleetDeployment(model_name="grok-4", name="grok-4", self_hosted=False))
sampler = MetricsSampler(fleet, HourlyRollup(15), interval_s=15)
# Read at the public port; named by the listen port. No address, no reading.
- assert sampler.targets() == [("203.0.113.10:8001", "qwen3-8b", "http://203.0.113.10:22018/metrics")]
+ assert sampler.targets() == [(reachable, "203.0.113.10:8001", "qwen3-8b", "http://203.0.113.10:22018/metrics")]
def test_a_pass_feeds_the_rollup_and_notes_a_box_that_did_not_answer():
@@ -134,3 +135,71 @@ def test_a_pass_feeds_the_rollup_and_notes_a_box_that_did_not_answer():
(hour,) = rollup.take_finished(time.time() + HOUR_S)
assert hour.server == "203.0.113.10:8001"
assert hour.readings == 1
+
+
+_CACHE_INFO = """\
+# HELP vllm:num_requests_running Number of requests in model execution batches.
+# TYPE vllm:num_requests_running gauge
+vllm:num_requests_running{model_name="google/gemma-4-26B-A4B-it"} 0.0
+# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig
+# TYPE vllm:cache_config_info gauge
+vllm:cache_config_info{block_size="16",cache_dtype="auto",enable_prefix_caching="True",gpu_memory_utilization="0.9",kv_cache_max_concurrency="6.52951355508556",kv_cache_memory_bytes="None",kv_cache_size_tokens="855836",num_cpu_blocks="None",num_gpu_blocks="33960"} 1.0
+"""
+
+
+def test_parse_kv_cache_tokens_prefers_the_reported_count():
+ """kv_cache_size_tokens is authoritative; the block product is not.
+
+ Real output from the A100 serving Gemma reports 855,836 tokens while
+ num_gpu_blocks * block_size gives 543,360 -- blocks do not map uniformly
+ to tokens for a model with heterogeneous head dimensions. Taking the
+ product would understate the cache by a third and falsely flag a
+ correctly-sized server as oversized.
+ """
+ assert parse_kv_cache_tokens(_CACHE_INFO) == 855836
+ assert 33960 * 16 == 543360 # what the wrong answer would have been
+
+
+def test_parse_kv_cache_tokens_falls_back_to_block_product():
+ """Builds that do not emit the token count still get an approximation."""
+ text = 'vllm:cache_config_info{block_size="16",num_gpu_blocks="1000"} 1.0'
+ assert parse_kv_cache_tokens(text) == 16000
+
+ text = 'vllm:cache_config_info{block_size="16",num_gpu_blocks="1000",kv_cache_size_tokens="None"} 1.0'
+ assert parse_kv_cache_tokens(text) == 16000
+
+
+def test_parse_kv_cache_tokens_absent():
+ """vLLM's metric names are not a stable API.
+
+ A missing or renamed metric degrades the display rather than breaking the
+ probe loop, so this returns None instead of raising.
+ """
+ assert parse_kv_cache_tokens("vllm:num_requests_running{} 0.0\n") is None
+ assert parse_kv_cache_tokens("") is None
+
+
+def test_parse_kv_cache_tokens_malformed():
+ for text in (
+ 'vllm:cache_config_info{block_size="16"} 1.0', # no block count
+ 'vllm:cache_config_info{block_size="x",num_gpu_blocks="1"} 1.0',
+ "vllm:cache_config_info block_size=16 1.0", # no braces
+ 'vllm:cache_config_info{block_size="0",num_gpu_blocks="0"} 1.0',
+ ):
+ assert parse_kv_cache_tokens(text) is None, text
+
+
+def test_a_reading_carries_the_kv_cache_size_and_the_sampler_hands_it_to_the_fleet():
+ assert parse_metrics(_CACHE_INFO, now=1.0).kv_cache_tokens == 855836
+ srv = _server("203.0.113.10")
+ fleet = Fleet(None, [srv])
+ sampler = MetricsSampler(fleet, HourlyRollup(15), interval_s=15)
+
+ async def one_pass():
+ transport = httpx.MockTransport(lambda request: httpx.Response(200, text=METRICS + _CACHE_INFO))
+ async with httpx.AsyncClient(transport=transport) as client:
+ await sampler.sample_once(client)
+
+ assert srv.kv_cache_tokens is None
+ asyncio.run(one_pass())
+ assert srv.kv_cache_tokens == 855836
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org