You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/03 20:44:14 UTC

(tooling-llmao) 04/04: status: record extra LiteLLM routes and model-name mismatches

This is an automated email from the ASF dual-hosted git repository.

gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git

commit 3031391850ecc74202895f3aa8f954026f9dd11e
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Sat Oct 3 15:35:21 2026 -0500

    status: record extra LiteLLM routes and model-name mismatches
    
    check_config_skew used to keep only the set of api_base values. An api_base
    with no deployment was a log line and then dropped. A matching api_base
    under a different model_name was not recorded at all.
    
    Each LiteLLM model/info row is now host, port, and model_name. api_base is
    not the key: one host serves several ports, and one host:port can report
    more than one model. Rows that match no deployment are rebuilt onto
    Fleet.extra_litellm each pass, with the in-LiteLLM-not-in-config badge.
    They are still not Fleet table rows.
    
    A deployment whose api_base is present under another model_name gets
    config_mismatch. snapshot_for_status copies that flag, so the badge
    survives the health pass, which replaces dep.skew from the snapshot. The
    same model_name clears it.
    
    Co-authored-by: Grok <gr...@x.ai>
---
 llmao/fleet.py             | 20 +++++++++++++++
 llmao/litellm_client.py    | 61 +++++++++++++++++++++++++++++++++++++---------
 tests/test_fleet_health.py | 42 +++++++++++++++++++++++++++++++
 3 files changed, 111 insertions(+), 12 deletions(-)

diff --git a/llmao/fleet.py b/llmao/fleet.py
index aa532f0..610d3cb 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -32,6 +32,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import time
+from dataclasses import dataclass
 from typing import Any
 
 import httpx
@@ -443,6 +444,8 @@ class FleetDeployment:
         self.litellm_healthy: bool | None = None
         self.litellm_health_at: float | None = None
         self.skew: list[str] = []
+        # Set by the config skew pass when LiteLLM has this api_base under another model_name.
+        self.config_mismatch = False
 
     @classmethod
     def from_vllm(cls, srv: VllmServer) -> FleetDeployment:
@@ -471,6 +474,16 @@ class FleetDeployment:
         return self._api_base
 
 
+@dataclass(frozen=True)
+class ExtraLiteLLM:
+    """One LiteLLM route that config.yaml does not list."""
+
+    host: str
+    port: int | None
+    model_name: str
+    skew: tuple[str, ...]
+
+
 class Fleet:
     """Live fleet: vLLM servers, intended deployments, lifecycle probes."""
 
@@ -494,6 +507,10 @@ class Fleet:
         self.config_fetch_at: dict[str, float] = {}
         # Fleet-key GETs for IPs not in fleet.hosts. Process memory only.
         self.unknown_config_fetches: dict[str, dict[str, float | int]] = {}
+        # LiteLLM routes that match no deployment. Rebuilt each config skew pass.
+        # Identity is host, port, and model name. api_base is not a primary key:
+        # one host serves several ports, and one host:port can report more than one model.
+        self.extra_litellm: list[ExtraLiteLLM] = []
 
     @classmethod
     def from_cfg(cls, cfg: Any, models: list | None = None) -> Fleet:
@@ -666,6 +683,7 @@ def snapshot_for_status(dep, cfg: edict, now: float) -> DeploymentSnapshot:
     misses = int(cfg.health_fail_threshold)
     grace = float(cfg.health_grace_s)
     litellm_at = now if dep.litellm_healthy is not None else None
+    mismatch = bool(dep.config_mismatch)
     srv = dep.vllm
     if srv is None:
         return DeploymentSnapshot(
@@ -674,6 +692,7 @@ def snapshot_for_status(dep, cfg: edict, now: float) -> DeploymentSnapshot:
             in_litellm=dep.in_litellm,
             litellm_healthy=dep.litellm_healthy,
             litellm_health_at=litellm_at,
+            config_mismatch=mismatch,
         )
     if srv.state == HEALTHY:
         probe_ok, fails, last_ok, seen, probed = True, 0, now, srv.seen_at, now
@@ -696,6 +715,7 @@ def snapshot_for_status(dep, cfg: edict, now: float) -> DeploymentSnapshot:
         litellm_health_at=litellm_at,
         context_window=srv.observed_max_model_len or srv.max_model_len,
         kv_cache_tokens=srv.kv_cache_tokens,
+        config_mismatch=mismatch,
     )
 
 
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index 5375778..06ca17c 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -31,11 +31,20 @@ import logging
 import time
 from dataclasses import dataclass
 from typing import Any, Protocol
+from urllib.parse import urlparse
 
 import httpx
 
-from llmao.fleet import snapshot_for_status
-from llmao.model_status import HEALTHY, SKEW_PHRASE, STALLED, UNHEALTHY, deployment_status
+from llmao.fleet import ExtraLiteLLM, snapshot_for_status
+from llmao.model_status import (
+    HEALTHY,
+    SKEW_IN_LITELLM_NOT_IN_CONFIG,
+    SKEW_PHRASE,
+    STALLED,
+    UNHEALTHY,
+    DeploymentSnapshot,
+    deployment_status,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -598,15 +607,35 @@ class LiteLLMBackend:
     async def check_config_skew(self) -> None:
         resp = await self._request("GET", "model/info")
         self._raise_http(resp)
-        bases = _api_bases_from_model_info(resp.json())
+        routes = _routes_from_model_info(resp.json())
+        by_base: dict[str, list[str]] = {}
+        for norm, _host, _port, model_name in routes:
+            by_base.setdefault(norm, []).append(model_name)
         intended = {_norm_base(d.api_base) for d in self.fleet.deployments if d.api_base}
+        now = time.time()
         for dep in self.fleet.deployments:
             base = _norm_base(dep.api_base) if dep.api_base else None
-            dep.in_litellm = bool(base) and base in bases
-            self._store_skew(dep, time.time())
-        extra = bases - intended
+            names = by_base.get(base, []) if base else []
+            dep.in_litellm = bool(names)
+            dep.config_mismatch = any(name != dep.model_name for name in names)
+            self._store_skew(dep, now)
+        extra: list[ExtraLiteLLM] = []
+        for norm, host, port, model_name in routes:
+            if norm in intended:
+                continue
+            view = deployment_status(
+                DeploymentSnapshot(model_name=model_name, self_hosted=False, litellm_only=True),
+                self._cfg.fleet,
+                now,
+            )
+            extra.append(ExtraLiteLLM(host=host, port=port, model_name=model_name, skew=view.skew))
+        self.fleet.extra_litellm = extra
         if extra:
-            _LOGGER.warning("skew: LiteLLM api_base not an intended deployment: %s", sorted(extra))
+            _LOGGER.warning(
+                "skew: %s %s",
+                SKEW_PHRASE[SKEW_IN_LITELLM_NOT_IN_CONFIG],
+                [(row.host, row.port, row.model_name) for row in extra],
+            )
 
     async def check_health_skew(self) -> None:
         resp = await self._request("GET", "health")
@@ -667,17 +696,25 @@ def _norm_base(url: Any) -> str:
     return s
 
 
-def _api_bases_from_model_info(body: Any) -> set[str]:
+def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str]]:
+    """Each LiteLLM row as (normalised api_base, host, port, model_name).
+
+    Host and port stay separate. One machine serves several ports, and one
+    host:port can still report more than one model name.
+    """
     rows = []
     if isinstance(body, list):
         rows = body
     elif isinstance(body, dict):
         rows = body.get("data") or body.get("models") or []
-    bases: set[str] = set()
+    out: list[tuple[str, str, int | None, str]] = []
     for row in rows:
         if not isinstance(row, dict):
             continue
         params = row.get("litellm_params") or {}
-        if isinstance(params, dict) and params.get("api_base"):
-            bases.add(_norm_base(params["api_base"]))
-    return bases
+        if not isinstance(params, dict) or not params.get("api_base"):
+            continue
+        norm = _norm_base(params["api_base"])
+        parsed = urlparse(norm)
+        out.append((norm, parsed.hostname or "", parsed.port, str(row.get("model_name") or "")))
+    return out
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index 145de61..94388aa 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -35,6 +35,8 @@ from llmao.model_status import (
     AVAILABLE,
     HEALTHY,
     LOADING,
+    SKEW_CONFIG_MISMATCH,
+    SKEW_IN_LITELLM_NOT_IN_CONFIG,
     SKEW_INTENDED_NOT_IN_LITELLM,
     SKEW_VLLM_DOWN_LITELLM_UP,
     SKEW_VLLM_UP_LITELLM_DOWN,
@@ -192,6 +194,46 @@ def test_skew_sentences_replace_old_phrases():
     assert "LiteLLM health disagrees" not in dep.skew
 
 
+def _skew_backend(fleet, info):
+    backend = _Backend(fleet, info=info, health={})
+    backend._cfg = fleet.cfg
+    backend._store_skew = lambda dep, now: LiteLLMBackend._store_skew(backend, dep, now)
+    return backend
+
+
+def test_extra_routes_are_per_port_and_model():
+    srv = _server()
+    dep = FleetDeployment.from_vllm(srv)
+    fleet = Fleet(cfg=EasyDict({"fleet": {"hosts": {}, **FLEET_KNOBS}}), servers=[srv], deployments=[dep])
+    info = [
+        {"model_name": "other-a", "litellm_params": {"api_base": "http://10.0.0.1:9001/v1"}},
+        {"model_name": "other-b", "litellm_params": {"api_base": "http://10.0.0.1:9002/v1"}},
+        {"model_name": "other-c", "litellm_params": {"api_base": "http://10.0.0.1:9002/v1"}},
+    ]
+    asyncio.run(LiteLLMBackend.check_config_skew(_skew_backend(fleet, info)))
+    found = {(row.host, row.port, row.model_name) for row in fleet.extra_litellm}
+    assert found == {("10.0.0.1", 9001, "other-a"), ("10.0.0.1", 9002, "other-b"), ("10.0.0.1", 9002, "other-c")}
+    assert all(SKEW_IN_LITELLM_NOT_IN_CONFIG in row.skew for row in fleet.extra_litellm)
+    assert SKEW_IN_LITELLM_NOT_IN_CONFIG not in dep.skew
+
+
+def test_model_name_mismatch_survives_health_check():
+    srv = _server()
+    srv.state = HEALTHY
+    dep = FleetDeployment.from_vllm(srv)
+    fleet = Fleet(cfg=EasyDict({"fleet": {"hosts": {}, **FLEET_KNOBS}}), servers=[srv], deployments=[dep])
+    backend = _skew_backend(fleet, [{"model_name": "not-gemma", "litellm_params": {"api_base": srv.api_base}}])
+    asyncio.run(LiteLLMBackend.check_config_skew(backend))
+    assert SKEW_CONFIG_MISMATCH in dep.skew
+    backend.health = {"healthy_endpoints": [{"api_base": srv.api_base}], "unhealthy_endpoints": []}
+    asyncio.run(LiteLLMBackend.check_health_skew(backend))
+    assert SKEW_CONFIG_MISMATCH in dep.skew
+    backend.info = [{"model_name": srv.model_name, "litellm_params": {"api_base": srv.api_base}}]
+    asyncio.run(LiteLLMBackend.check_config_skew(backend))
+    assert SKEW_CONFIG_MISMATCH not in dep.skew
+    assert fleet.extra_litellm == []
+
+
 def test_add_calls_deployment_only_when_serving():
     srv = _server()
     dep = FleetDeployment.from_vllm(srv)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org