You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/03 20:44:14 UTC
(tooling-llmao) 04/04: status: record extra LiteLLM routes and model-name mismatches
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 3031391850ecc74202895f3aa8f954026f9dd11e
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Sat Oct 3 15:35:21 2026 -0500
status: record extra LiteLLM routes and model-name mismatches
check_config_skew used to keep only the set of api_base values. An api_base
with no deployment was a log line and then dropped. A matching api_base
under a different model_name was not recorded at all.
Each LiteLLM model/info row is now host, port, and model_name. api_base is
not the key: one host serves several ports, and one host:port can report
more than one model. Rows that match no deployment are rebuilt onto
Fleet.extra_litellm each pass, with the in-LiteLLM-not-in-config badge.
They are still not Fleet table rows.
A deployment whose api_base is present under another model_name gets
config_mismatch. snapshot_for_status copies that flag, so the badge
survives the health pass, which replaces dep.skew from the snapshot. The
same model_name clears it.
Co-authored-by: Grok <gr...@x.ai>
---
llmao/fleet.py | 20 +++++++++++++++
llmao/litellm_client.py | 61 +++++++++++++++++++++++++++++++++++++---------
tests/test_fleet_health.py | 42 +++++++++++++++++++++++++++++++
3 files changed, 111 insertions(+), 12 deletions(-)
diff --git a/llmao/fleet.py b/llmao/fleet.py
index aa532f0..610d3cb 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -32,6 +32,7 @@ from __future__ import annotations
import asyncio
import logging
import time
+from dataclasses import dataclass
from typing import Any
import httpx
@@ -443,6 +444,8 @@ class FleetDeployment:
self.litellm_healthy: bool | None = None
self.litellm_health_at: float | None = None
self.skew: list[str] = []
+ # Set by the config skew pass when LiteLLM has this api_base under another model_name.
+ self.config_mismatch = False
@classmethod
def from_vllm(cls, srv: VllmServer) -> FleetDeployment:
@@ -471,6 +474,16 @@ class FleetDeployment:
return self._api_base
+@dataclass(frozen=True)
+class ExtraLiteLLM:
+ """One LiteLLM route that config.yaml does not list."""
+
+ host: str
+ port: int | None
+ model_name: str
+ skew: tuple[str, ...]
+
+
class Fleet:
"""Live fleet: vLLM servers, intended deployments, lifecycle probes."""
@@ -494,6 +507,10 @@ class Fleet:
self.config_fetch_at: dict[str, float] = {}
# Fleet-key GETs for IPs not in fleet.hosts. Process memory only.
self.unknown_config_fetches: dict[str, dict[str, float | int]] = {}
+ # LiteLLM routes that match no deployment. Rebuilt each config skew pass.
+ # Identity is host, port, and model name. api_base is not a primary key:
+ # one host serves several ports, and one host:port can report more than one model.
+ self.extra_litellm: list[ExtraLiteLLM] = []
@classmethod
def from_cfg(cls, cfg: Any, models: list | None = None) -> Fleet:
@@ -666,6 +683,7 @@ def snapshot_for_status(dep, cfg: edict, now: float) -> DeploymentSnapshot:
misses = int(cfg.health_fail_threshold)
grace = float(cfg.health_grace_s)
litellm_at = now if dep.litellm_healthy is not None else None
+ mismatch = bool(dep.config_mismatch)
srv = dep.vllm
if srv is None:
return DeploymentSnapshot(
@@ -674,6 +692,7 @@ def snapshot_for_status(dep, cfg: edict, now: float) -> DeploymentSnapshot:
in_litellm=dep.in_litellm,
litellm_healthy=dep.litellm_healthy,
litellm_health_at=litellm_at,
+ config_mismatch=mismatch,
)
if srv.state == HEALTHY:
probe_ok, fails, last_ok, seen, probed = True, 0, now, srv.seen_at, now
@@ -696,6 +715,7 @@ def snapshot_for_status(dep, cfg: edict, now: float) -> DeploymentSnapshot:
litellm_health_at=litellm_at,
context_window=srv.observed_max_model_len or srv.max_model_len,
kv_cache_tokens=srv.kv_cache_tokens,
+ config_mismatch=mismatch,
)
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index 5375778..06ca17c 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -31,11 +31,20 @@ import logging
import time
from dataclasses import dataclass
from typing import Any, Protocol
+from urllib.parse import urlparse
import httpx
-from llmao.fleet import snapshot_for_status
-from llmao.model_status import HEALTHY, SKEW_PHRASE, STALLED, UNHEALTHY, deployment_status
+from llmao.fleet import ExtraLiteLLM, snapshot_for_status
+from llmao.model_status import (
+ HEALTHY,
+ SKEW_IN_LITELLM_NOT_IN_CONFIG,
+ SKEW_PHRASE,
+ STALLED,
+ UNHEALTHY,
+ DeploymentSnapshot,
+ deployment_status,
+)
_LOGGER = logging.getLogger(__name__)
@@ -598,15 +607,35 @@ class LiteLLMBackend:
async def check_config_skew(self) -> None:
resp = await self._request("GET", "model/info")
self._raise_http(resp)
- bases = _api_bases_from_model_info(resp.json())
+ routes = _routes_from_model_info(resp.json())
+ by_base: dict[str, list[str]] = {}
+ for norm, _host, _port, model_name in routes:
+ by_base.setdefault(norm, []).append(model_name)
intended = {_norm_base(d.api_base) for d in self.fleet.deployments if d.api_base}
+ now = time.time()
for dep in self.fleet.deployments:
base = _norm_base(dep.api_base) if dep.api_base else None
- dep.in_litellm = bool(base) and base in bases
- self._store_skew(dep, time.time())
- extra = bases - intended
+ names = by_base.get(base, []) if base else []
+ dep.in_litellm = bool(names)
+ dep.config_mismatch = any(name != dep.model_name for name in names)
+ self._store_skew(dep, now)
+ extra: list[ExtraLiteLLM] = []
+ for norm, host, port, model_name in routes:
+ if norm in intended:
+ continue
+ view = deployment_status(
+ DeploymentSnapshot(model_name=model_name, self_hosted=False, litellm_only=True),
+ self._cfg.fleet,
+ now,
+ )
+ extra.append(ExtraLiteLLM(host=host, port=port, model_name=model_name, skew=view.skew))
+ self.fleet.extra_litellm = extra
if extra:
- _LOGGER.warning("skew: LiteLLM api_base not an intended deployment: %s", sorted(extra))
+ _LOGGER.warning(
+ "skew: %s %s",
+ SKEW_PHRASE[SKEW_IN_LITELLM_NOT_IN_CONFIG],
+ [(row.host, row.port, row.model_name) for row in extra],
+ )
async def check_health_skew(self) -> None:
resp = await self._request("GET", "health")
@@ -667,17 +696,25 @@ def _norm_base(url: Any) -> str:
return s
-def _api_bases_from_model_info(body: Any) -> set[str]:
+def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str]]:
+ """Each LiteLLM row as (normalised api_base, host, port, model_name).
+
+ Host and port stay separate. One machine serves several ports, and one
+ host:port can still report more than one model name.
+ """
rows = []
if isinstance(body, list):
rows = body
elif isinstance(body, dict):
rows = body.get("data") or body.get("models") or []
- bases: set[str] = set()
+ out: list[tuple[str, str, int | None, str]] = []
for row in rows:
if not isinstance(row, dict):
continue
params = row.get("litellm_params") or {}
- if isinstance(params, dict) and params.get("api_base"):
- bases.add(_norm_base(params["api_base"]))
- return bases
+ if not isinstance(params, dict) or not params.get("api_base"):
+ continue
+ norm = _norm_base(params["api_base"])
+ parsed = urlparse(norm)
+ out.append((norm, parsed.hostname or "", parsed.port, str(row.get("model_name") or "")))
+ return out
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index 145de61..94388aa 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -35,6 +35,8 @@ from llmao.model_status import (
AVAILABLE,
HEALTHY,
LOADING,
+ SKEW_CONFIG_MISMATCH,
+ SKEW_IN_LITELLM_NOT_IN_CONFIG,
SKEW_INTENDED_NOT_IN_LITELLM,
SKEW_VLLM_DOWN_LITELLM_UP,
SKEW_VLLM_UP_LITELLM_DOWN,
@@ -192,6 +194,46 @@ def test_skew_sentences_replace_old_phrases():
assert "LiteLLM health disagrees" not in dep.skew
+def _skew_backend(fleet, info):
+ backend = _Backend(fleet, info=info, health={})
+ backend._cfg = fleet.cfg
+ backend._store_skew = lambda dep, now: LiteLLMBackend._store_skew(backend, dep, now)
+ return backend
+
+
+def test_extra_routes_are_per_port_and_model():
+ srv = _server()
+ dep = FleetDeployment.from_vllm(srv)
+ fleet = Fleet(cfg=EasyDict({"fleet": {"hosts": {}, **FLEET_KNOBS}}), servers=[srv], deployments=[dep])
+ info = [
+ {"model_name": "other-a", "litellm_params": {"api_base": "http://10.0.0.1:9001/v1"}},
+ {"model_name": "other-b", "litellm_params": {"api_base": "http://10.0.0.1:9002/v1"}},
+ {"model_name": "other-c", "litellm_params": {"api_base": "http://10.0.0.1:9002/v1"}},
+ ]
+ asyncio.run(LiteLLMBackend.check_config_skew(_skew_backend(fleet, info)))
+ found = {(row.host, row.port, row.model_name) for row in fleet.extra_litellm}
+ assert found == {("10.0.0.1", 9001, "other-a"), ("10.0.0.1", 9002, "other-b"), ("10.0.0.1", 9002, "other-c")}
+ assert all(SKEW_IN_LITELLM_NOT_IN_CONFIG in row.skew for row in fleet.extra_litellm)
+ assert SKEW_IN_LITELLM_NOT_IN_CONFIG not in dep.skew
+
+
+def test_model_name_mismatch_survives_health_check():
+ srv = _server()
+ srv.state = HEALTHY
+ dep = FleetDeployment.from_vllm(srv)
+ fleet = Fleet(cfg=EasyDict({"fleet": {"hosts": {}, **FLEET_KNOBS}}), servers=[srv], deployments=[dep])
+ backend = _skew_backend(fleet, [{"model_name": "not-gemma", "litellm_params": {"api_base": srv.api_base}}])
+ asyncio.run(LiteLLMBackend.check_config_skew(backend))
+ assert SKEW_CONFIG_MISMATCH in dep.skew
+ backend.health = {"healthy_endpoints": [{"api_base": srv.api_base}], "unhealthy_endpoints": []}
+ asyncio.run(LiteLLMBackend.check_health_skew(backend))
+ assert SKEW_CONFIG_MISMATCH in dep.skew
+ backend.info = [{"model_name": srv.model_name, "litellm_params": {"api_base": srv.api_base}}]
+ asyncio.run(LiteLLMBackend.check_config_skew(backend))
+ assert SKEW_CONFIG_MISMATCH not in dep.skew
+ assert fleet.extra_litellm == []
+
+
def test_add_calls_deployment_only_when_serving():
srv = _server()
dep = FleetDeployment.from_vllm(srv)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org