You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/13 00:03:42 UTC
(tooling-llmao) branch main updated (8b592f5 -> 8b04323)
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
from 8b592f5 client: give the HTTP plumbing a class of its own, and tests a stand-in
new 62c5b33 fleet: a deployment knows whether its cost is capacity or money
new d18b9d2 proxy: move litellm_client.py to llmao/proxy/client.py
new 8b04323 proxy: DeploymentManager takes the deployment code out of the backend
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
llmao/fleet.py | 30 ++-
llmao/{ => proxy}/__init__.py | 9 +-
llmao/{litellm_client.py => proxy/client.py} | 354 +-------------------------
llmao/proxy/deployments.py | 368 +++++++++++++++++++++++++++
llmao/{__init__.py => proxy/errors.py} | 17 +-
llmao/seam.py | 2 +-
main.py | 6 +-
pages.py | 6 +-
tests/fake_transport.py | 2 +-
tests/mock_backend.py | 2 +-
tests/test_access_requests.py | 4 +-
tests/test_admin.py | 2 +-
tests/test_entitlements.py | 2 +-
tests/test_fleet.py | 10 +-
tests/test_fleet_health.py | 80 +++---
tests/test_mix.py | 128 ++--------
tests/test_projects.py | 2 +-
tests/test_spend_rows.py | 31 ++-
19 files changed, 524 insertions(+), 533 deletions(-)
copy llmao/{ => proxy}/__init__.py (72%)
rename llmao/{litellm_client.py => proxy/client.py} (77%)
create mode 100644 llmao/proxy/deployments.py
copy llmao/{__init__.py => proxy/errors.py} (63%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-llmao) 02/03: proxy: move litellm_client.py to llmao/proxy/client.py
Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit d18b9d2a4048fc77680146aaef54747a842e9799
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Mon Oct 12 18:47:45 2026 -0500
proxy: move litellm_client.py to llmao/proxy/client.py
No functional change. Name and packaging only.
llmao/litellm_client.py becomes llmao/proxy/client.py, in a new package
that the pieces being split out of it will join. "Proxy" is LiteLLM's name
for its server; the package is our client for it, not a proxy of our own.
Every import of the old path is updated. The one line inside the file that
changes is the path to pyproject.toml in pinned_litellm_version, which is
now one directory further up.
Log lines from this module are now labelled llmao.proxy.client.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
README.md | 2 +-
llmao/proxy/__init__.py | 22 ++++++++++++++++++++++
llmao/{litellm_client.py => proxy/client.py} | 2 +-
llmao/seam.py | 2 +-
main.py | 2 +-
pages.py | 2 +-
tests/fake_transport.py | 2 +-
tests/mock_backend.py | 2 +-
tests/test_access_requests.py | 2 +-
tests/test_admin.py | 2 +-
tests/test_entitlements.py | 2 +-
tests/test_fleet.py | 4 ++--
tests/test_fleet_health.py | 2 +-
tests/test_mix.py | 2 +-
tests/test_projects.py | 2 +-
tests/test_spend_rows.py | 2 +-
16 files changed, 38 insertions(+), 16 deletions(-)
diff --git a/README.md b/README.md
index 02d00c7..1e52385 100644
--- a/README.md
+++ b/README.md
@@ -229,7 +229,7 @@ config.yaml.example → config.yaml (gitignored; secrets)
litellm.yaml.example → litellm.yaml (do not include models.yaml; store_model_in_db)
models.yaml admin definitions (committed; no secrets)
certs/ mkcert PEMs + README
-llmao/ seam, auth, models, litellm_client, fleet
+llmao/ seam, auth, models, fleet, proxy/ (LiteLLM client)
hosting/vast/ provision.sh + install_set.py
hosting/runpod/ RunPod template notes (image not built yet)
tests/ offline seam, fleet, hosting installer
diff --git a/llmao/proxy/__init__.py b/llmao/proxy/__init__.py
new file mode 100644
index 0000000..37c85fd
--- /dev/null
+++ b/llmao/proxy/__init__.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""LLMAO's client for the LiteLLM proxy.
+
+Not a proxy of its own. LiteLLM calls its server the proxy; this package is
+everything here that talks to its admin API.
+"""
diff --git a/llmao/litellm_client.py b/llmao/proxy/client.py
similarity index 99%
rename from llmao/litellm_client.py
rename to llmao/proxy/client.py
index 5259480..005296e 100644
--- a/llmao/litellm_client.py
+++ b/llmao/proxy/client.py
@@ -83,7 +83,7 @@ _LITELLM_PIN = re.compile(r"""litellm(?:\[[^\]]+\])?==([^"'\s]+)""")
def pinned_litellm_version() -> str:
"""The litellm version pyproject.toml requires. The admin page compares the proxy to this."""
- text = (pathlib.Path(__file__).resolve().parents[1] / "pyproject.toml").read_text(encoding="utf-8")
+ text = (pathlib.Path(__file__).resolve().parents[2] / "pyproject.toml").read_text(encoding="utf-8")
match = _LITELLM_PIN.search(text)
if match is None:
raise RuntimeError("pyproject.toml has no litellm pin")
diff --git a/llmao/seam.py b/llmao/seam.py
index 859a961..110f97f 100644
--- a/llmao/seam.py
+++ b/llmao/seam.py
@@ -31,7 +31,7 @@ from dataclasses import dataclass, field
from typing import Any
from . import access_requests as _access
-from .litellm_client import (
+from .proxy.client import (
GRANTOR_FREE_TIER,
TIER_FREE,
TIER_PROJECT,
diff --git a/main.py b/main.py
index 3df9da2..e42c9ca 100644
--- a/main.py
+++ b/main.py
@@ -70,10 +70,10 @@ def create_app():
# Fail-fast: models.yaml is required (same presumption as config.yaml).
from llmao.contention import ContentionSampler
from llmao.fleet import Fleet, validate_fleet
- from llmao.litellm_client import LiteLLMBackend
# Config is app.cfg (EasyDict from config.yaml) — dotted access throughout.
from llmao.models import load_models
+ from llmao.proxy.client import LiteLLMBackend
from llmao.seam import Seam
definitions = load_models(cfg=app.cfg)
diff --git a/pages.py b/pages.py
index 50ae30e..d056a2e 100644
--- a/pages.py
+++ b/pages.py
@@ -41,7 +41,6 @@ from easydict import EasyDict as edict # noqa: N813
from llmao.auth import current_identity
from llmao.fleet import add_refusal, remove_refusal, snapshot_for_status
-from llmao.litellm_client import BackendUnavailableError, KeyInfo, pinned_litellm_version
from llmao.model_status import (
ADMIN_LABELS,
AVAILABLE,
@@ -61,6 +60,7 @@ from llmao.model_status import (
deployment_status,
)
from llmao.models import load_models, model_available_for, model_in_service, ux_models
+from llmao.proxy.client import BackendUnavailableError, KeyInfo, pinned_litellm_version
from llmao.seam import AuthzError
# Replica summary order: the calm states first, the ones needing attention last.
diff --git a/tests/fake_transport.py b/tests/fake_transport.py
index 9b2e3d3..028525e 100644
--- a/tests/fake_transport.py
+++ b/tests/fake_transport.py
@@ -32,7 +32,7 @@ from __future__ import annotations
from easydict import EasyDict as edict # noqa: N813
-from llmao.litellm_client import BackendUnavailableError
+from llmao.proxy.client import BackendUnavailableError
class FakeTransport:
diff --git a/tests/mock_backend.py b/tests/mock_backend.py
index ac801bd..56120b9 100644
--- a/tests/mock_backend.py
+++ b/tests/mock_backend.py
@@ -27,7 +27,7 @@ import time
import uuid
from typing import Any
-from llmao.litellm_client import (
+from llmao.proxy.client import (
GRANTOR_FREE_TIER,
CreatedKey,
KeyInfo,
diff --git a/tests/test_access_requests.py b/tests/test_access_requests.py
index c70a533..c50fdba 100644
--- a/tests/test_access_requests.py
+++ b/tests/test_access_requests.py
@@ -44,7 +44,7 @@ from llmao.access_requests import (
loads,
prune,
)
-from llmao.litellm_client import LiteLLMBackend
+from llmao.proxy.client import LiteLLMBackend
UTC = dt.UTC
diff --git a/tests/test_admin.py b/tests/test_admin.py
index afdc586..502f8b4 100644
--- a/tests/test_admin.py
+++ b/tests/test_admin.py
@@ -25,7 +25,7 @@ import ezt
import httpx
from easydict import EasyDict as edict # noqa: N813
-from llmao.litellm_client import LiteLLMBackend, LiteLLMTransport, pinned_litellm_version
+from llmao.proxy.client import LiteLLMBackend, LiteLLMTransport, pinned_litellm_version
THIS_DIR = pathlib.Path(__file__).resolve().parent.parent
diff --git a/tests/test_entitlements.py b/tests/test_entitlements.py
index 37646c1..c4f0fcc 100644
--- a/tests/test_entitlements.py
+++ b/tests/test_entitlements.py
@@ -28,7 +28,7 @@ import datetime as dt
import pytest
from easydict import EasyDict as edict # noqa: N813
-from llmao.litellm_client import TIER_FREE, TIER_PROJECT, TIER_SERVICE, resolve_entitlement
+from llmao.proxy.client import TIER_FREE, TIER_PROJECT, TIER_SERVICE, resolve_entitlement
from llmao.token_window import RollingAllowance
UTC = dt.UTC
diff --git a/tests/test_fleet.py b/tests/test_fleet.py
index 45d3252..b30522a 100644
--- a/tests/test_fleet.py
+++ b/tests/test_fleet.py
@@ -36,9 +36,9 @@ from llmao.fleet import (
snapshot_for_status,
validate_fleet,
)
-from llmao.litellm_client import _norm_base
from llmao.model_status import AWAITING, HEALTHY, LOADING, deployment_status
from llmao.models import load_models
+from llmao.proxy.client import _norm_base
from llmao.vllm_api_key import derive_vllm_api_key
EXAMPLE = Path(__file__).resolve().parent.parent / "models.yaml"
@@ -377,7 +377,7 @@ def test_legacy_boxes_need_the_shared_key_and_keep_their_ports():
def test_working_api_key_follows_the_legacy_set():
- from llmao.litellm_client import LiteLLMBackend
+ from llmao.proxy.client import LiteLLMBackend
cfg = _cfg({"80.188.223.202": [["gemma4-26b", 10100]]})
cfg.fleet.selfhost_api_key = "sk-shared"
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index f55166a..898c7bb 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -31,7 +31,6 @@ from llmao.fleet import (
parse_kv_cache_tokens,
parse_max_model_len,
)
-from llmao.litellm_client import LiteLLMBackend
from llmao.model_status import (
AVAILABLE,
AWAITING,
@@ -46,6 +45,7 @@ from llmao.model_status import (
UNAVAILABLE,
UNHEALTHY,
)
+from llmao.proxy.client import LiteLLMBackend
from tests.test_fleet import FLEET_KNOBS
diff --git a/tests/test_mix.py b/tests/test_mix.py
index 9e2204e..533a1a7 100644
--- a/tests/test_mix.py
+++ b/tests/test_mix.py
@@ -22,8 +22,8 @@ import asyncio
from easydict import EasyDict as edict # noqa: N813
from llmao.fleet import Fleet, FleetDeployment
-from llmao.litellm_client import LiteLLMBackend
from llmao.model_status import HEALTHY, UNHEALTHY
+from llmao.proxy.client import LiteLLMBackend
from tests.test_fleet_health import _server
diff --git a/tests/test_projects.py b/tests/test_projects.py
index 02cfdde..c46d37b 100644
--- a/tests/test_projects.py
+++ b/tests/test_projects.py
@@ -22,7 +22,7 @@ import asyncio
import pytest
from easydict import EasyDict as edict # noqa: N813
-from llmao.litellm_client import GRANTOR_FREE_TIER, BackendUnavailableError, resolve_budget_duration
+from llmao.proxy.client import GRANTOR_FREE_TIER, BackendUnavailableError, resolve_budget_duration
from llmao.seam import (
AuthzError,
Identity,
diff --git a/tests/test_spend_rows.py b/tests/test_spend_rows.py
index fc94223..175a0cb 100644
--- a/tests/test_spend_rows.py
+++ b/tests/test_spend_rows.py
@@ -28,7 +28,7 @@ from easydict import EasyDict as edict # noqa: N813
from fake_transport import FakeTransport
from llmao.fleet import Fleet, FleetDeployment
-from llmao.litellm_client import LiteLLMBackend
+from llmao.proxy.client import LiteLLMBackend
NOON = dt.datetime(2026, 10, 12, 12, 0, tzinfo=dt.UTC)
TODAY = "2026-10-12"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-llmao) 03/03: proxy: DeploymentManager takes the deployment code out of the backend
Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 8b04323632a5a5156cca67b09d9e28502b071c4e
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Mon Oct 12 19:02:41 2026 -0500
proxy: DeploymentManager takes the deployment code out of the backend
The second piece out of what was one 1,600-line class. Registering and
removing LiteLLM deployments, the sync that follows the fleet's health, and
the skew loop move to llmao/proxy/deployments.py as DeploymentManager.
The methods are the ones the backend had, in the same order. The only
change inside them is that they call the transport directly rather than
through the backend's _request and _raise_http. The manager is handed the
config, the fleet and the transport, and keeps no state: what it learns is
still written onto the fleet's FleetDeployment records.
Its four callers use it directly -- backend.deployments.add_deployment and
so on, in pages.py and main.py -- rather than through forwarding methods on
the backend.
llmao/proxy/errors.py holds the two exception classes, which the client and
the manager both raise.
test_mix and test_fleet_health build a real DeploymentManager over
FakeTransport. The five hand-written response classes and the stand-in
backend the skew checks were called against are gone.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
llmao/fleet.py | 2 +-
llmao/proxy/client.py | 335 +-------------------------------------
llmao/proxy/deployments.py | 368 ++++++++++++++++++++++++++++++++++++++++++
llmao/proxy/errors.py | 30 ++++
main.py | 4 +-
pages.py | 4 +-
tests/test_access_requests.py | 2 +-
tests/test_fleet.py | 8 +-
tests/test_fleet_health.py | 80 ++++-----
tests/test_mix.py | 128 +++------------
tests/test_spend_rows.py | 2 +-
11 files changed, 468 insertions(+), 495 deletions(-)
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 913d92e..e7d7d17 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -638,7 +638,7 @@ class Fleet:
# model_name → models.yaml row. POST /model/new copies litellm_params
# from here; LiteLLM /model/info encrypts them on the way back.
self.models = models if models is not None else {}
- # Awaitable after each vLLM probe tick (LiteLLMBackend.sync_selfhost).
+ # Awaitable after each vLLM probe tick (DeploymentManager.sync_selfhost).
# Wired from create_app so this module does not import the LiteLLM client.
self.after_probe = after_probe
self.config_fetch_at: dict[str, float] = {}
diff --git a/llmao/proxy/client.py b/llmao/proxy/client.py
index 005296e..03d8668 100644
--- a/llmao/proxy/client.py
+++ b/llmao/proxy/client.py
@@ -26,7 +26,6 @@ LiteLLM opaque team_id is internal to LiteLLMBackend only.
from __future__ import annotations
-import asyncio
import datetime as _dt
import json
import logging
@@ -35,35 +34,18 @@ import re
import time
from dataclasses import dataclass, field
from typing import Any, Protocol
-from urllib.parse import urlparse
import httpx
from asfpy.stopwatch import Stopwatch
from llmao import access_requests as _access
-from llmao.fleet import LEGACY_SERVERS, ExtraLiteLLM, normalize_peer_ip, snapshot_for_status
-from llmao.model_status import (
- HEALTHY,
- SKEW_IN_LITELLM_NOT_IN_CONFIG,
- SKEW_PHRASE,
- STALLED,
- UNHEALTHY,
- DeploymentSnapshot,
- deployment_status,
-)
+from llmao.proxy.deployments import DeploymentManager
+from llmao.proxy.errors import BackendUnavailableError
from llmao.token_window import RollingAllowance
_LOGGER = logging.getLogger(__name__)
-class BudgetExceededError(Exception):
- """Raised when a team is over budget (mirrors litellm proxy's 4xx)."""
-
-
-class BackendUnavailableError(Exception):
- """Raised when the litellm admin API times out or can't be reached."""
-
-
# Who authorized the project's dollar ceiling. First cfg default → Free Tier.
GRANTOR_FREE_TIER = "Free Tier"
@@ -387,6 +369,7 @@ class LiteLLMBackend:
self._cfg = cfg
self.fleet = fleet
self._transport = transport or LiteLLMTransport(cfg)
+ self.deployments = DeploymentManager(cfg, fleet, self._transport)
self._team_ids: dict[str, str] = {} # project → team_id
# ISO date → usage rows for that UTC day, from LiteLLM's daily
# aggregate. Spend and allowances are both derived from this.
@@ -493,7 +476,7 @@ class LiteLLMBackend:
"""Load project→team_id from LiteLLM. Fail-fast if the proxy is unreachable."""
rows = await self._team_list_rows(fresh=True)
await self._backfill_token_caps(rows)
- await self.ensure_commercial()
+ await self.deployments.ensure_commercial()
# Six of the week's seven days are history and are never asked for
# again, so reading them here means no visitor waits for them. Not
# worth refusing to start over: a cold cache is only slow.
@@ -1289,313 +1272,3 @@ class LiteLLMBackend:
not a minute old when they look at it.
"""
self._daily_at = None
-
- def deployment_body(self, dep, *, api_key: str | None = None) -> dict:
- """POST /model/new payload from models.yaml + this deployment's api_base.
-
- `api_key` overrides the deployment's own. Callers that can probe the
- box pass what it actually accepts; see _working_api_key.
- """
- entry = self.fleet.models.get(dep.model_name)
- if entry is None:
- raise BackendUnavailableError(f"models.yaml missing {dep.model_name}; cannot POST /model/new")
- params = dict(entry.litellm_params)
- if not dep.api_base:
- raise BackendUnavailableError(f"{dep.name}: no api_base for /model/new")
- params["api_base"] = dep.api_base
- if dep.self_hosted and dep.vllm is not None and dep.vllm.api_key:
- params["api_key"] = api_key or dep.vllm.api_key
-
- info: dict[str, Any] = {
- "self_hosted": dep.self_hosted,
- "asf_api_base": _norm_base(dep.api_base),
- }
-
- # Price self-hosted tokens so usage reporting shows something other
- # than 0.00. LiteLLM has no cost-map entry for these models, so every
- # completion prices at nothing and a project's draw is invisible.
- #
- # The price is derived per model in models.yaml from measured
- # throughput: hourly rental over tokens per second, input and output
- # separately because prefill runs at thousands per second and decode
- # at tens. On this fleet that is a 12-27x ratio, close to what
- # commercial providers charge.
- #
- # It follows the card rather than being one fleet-wide figure, because
- # a faster box costs more per hour and should cost more per token --
- # otherwise a model's price stops tracking what serving it costs.
- #
- # IT IS NOT A MARGINAL COST. The boxes are rented continuously, so a
- # token costs the same whether or not anyone asks for it, and the
- # figure above assumes a box decoding flat out. At 25% utilisation the
- # real cost is four times this; at 5%, twenty times. Read it as a unit
- # of fair share that happens to carry a dollar sign -- useful for
- # spotting a runaway job and comparing one project's draw against
- # another's, and wrong the moment anyone treats it as money the
- # Foundation could have kept.
- #
- # The UI shows it alongside a token count so the distinction survives
- # contact with a reader.
- if dep.self_hosted:
- pricing = getattr(entry, "pricing", None)
- if pricing is not None:
- for name in ("input_cost_per_token", "output_cost_per_token"):
- val = getattr(pricing, name, None)
- if val is not None:
- info[name] = float(val)
- # Marks the row as capacity rather than money, so a report can
- # keep the two apart instead of adding them together.
- info["cost_basis"] = dep.cost_basis
-
- return {
- "model_name": dep.model_name,
- "litellm_params": params,
- "model_info": info,
- }
-
- async def _model_info_rows(self) -> list:
- resp = await self._request("GET", "model/info")
- self._raise_http(resp)
- body = resp.json()
- if isinstance(body, list):
- return body
- if isinstance(body, dict):
- return body.get("data") or body.get("models") or []
- return []
-
- def _litellm_id(self, row: dict, dep) -> str | None:
- if str(row.get("model_name") or "") != dep.model_name:
- return None
- want = _norm_base(dep.api_base)
- info = row.get("model_info") or {}
- if not isinstance(info, dict):
- info = {}
- if (
- _norm_base(info.get("asf_api_base")) == want
- or _norm_base(
- (row.get("litellm_params") or {}).get("api_base") if isinstance(row.get("litellm_params"), dict) else ""
- )
- == want
- ):
- return str(info.get("id") or row.get("model_id") or "") or None
- return None
-
- async def _working_api_key(self, dep) -> str | None:
- """The bearer to register for this server.
-
- LEGACY_SERVERS were launched with the shared selfhost_api_key.
- validate_fleet already required that key when one of those hosts is
- configured, so this just returns it. Every other box uses the key
- derived from the salt. No probe: trying the wrong one 401s and the
- next pass deletes the route.
- """
- if not dep.self_hosted or dep.vllm is None:
- return None
- srv = dep.vllm
- if (normalize_peer_ip(srv.host), int(srv.listen_port)) in LEGACY_SERVERS:
- return str(self._cfg.fleet.selfhost_api_key).strip()
- return srv.api_key or None
-
- async def _deployment_present(self, dep) -> bool:
- if not dep.self_hosted or dep.vllm is None:
- return False
- for _norm, host, port, model_name in _routes_from_model_info(await self._model_info_rows()):
- if dep.vllm is not None and dep.vllm.same_deployment(host, port, model_name):
- return True
- return False
-
- async def add_deployment(self, dep) -> None:
- """POST /model/new for this deployment. No serving check.
-
- Callers decide the gate. sync_selfhost waits until vLLM is SERVING.
- POST /do-add-deployment uses add_refusal for the same gate, then calls
- this. A deployment already present for this server, on the listen port
- or the public port, is not posted again. A 400 or 409 means the same thing.
- """
- if await self._deployment_present(dep):
- dep.in_litellm = True
- _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
- return
- body = self.deployment_body(dep, api_key=await self._working_api_key(dep))
- resp = await self._request("POST", "model/new", json=body)
- if resp.status_code in (400, 409):
- dep.in_litellm = True
- _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
- return
- self._raise_http(resp)
- dep.in_litellm = True
- _LOGGER.info("model/new %s@%s", dep.name, dep.api_base)
-
- async def delete_deployment(self, dep) -> None:
- rows = await self._model_info_rows()
- found = None
- for row in rows:
- if isinstance(row, dict):
- found = self._litellm_id(row, dep)
- if found:
- break
- if not found:
- dep.in_litellm = False
- _LOGGER.warning("model/delete: no LiteLLM id for %s@%s", dep.name, dep.api_base)
- return
- resp = await self._request("POST", "model/delete", json={"id": found})
- self._raise_http(resp)
- dep.in_litellm = False
- _LOGGER.info("model/delete %s@%s id=%s", dep.name, dep.api_base, found)
-
- async def ensure_commercial(self) -> None:
- """At startup: POST /model/new for each commercial models.yaml deployment."""
- for dep in self.fleet.deployments:
- if dep.self_hosted or not dep.api_base:
- continue
- if dep.in_litellm:
- continue
- await self.add_deployment(dep)
-
- async def sync_selfhost(self) -> None:
- """After vLLM probes: register serving backends, drop down ones."""
- for dep in self.fleet.deployments:
- if not dep.self_hosted or dep.vllm is None:
- continue
- srv = dep.vllm
- if srv.state == HEALTHY and dep.api_base and not dep.in_litellm:
- await self.add_deployment(dep)
- elif srv.state in (UNHEALTHY, STALLED) and dep.in_litellm:
- await self.delete_deployment(dep)
-
- async def run_skew(self) -> None:
- """Config skew often; LiteLLM GET /health (tokens) on a long interval."""
- skew_s = float(self._cfg.fleet.skew_interval_s)
- health_s = float(self._cfg.fleet.litellm_health_interval_s)
- last_health = 0.0
- first = True
- while True:
- try:
- await self.check_config_skew()
- now = time.time()
- if first or (now - last_health) >= health_s:
- await self.check_health_skew()
- last_health = now
- first = False
- except Exception:
- _LOGGER.exception("LiteLLM skew check failed")
- await asyncio.sleep(skew_s)
-
- async def check_config_skew(self) -> None:
- resp = await self._request("GET", "model/info")
- self._raise_http(resp)
- routes = _routes_from_model_info(resp.json())
- by_base: dict[str, list[str]] = {}
- for norm, _host, _port, model_name in routes:
- by_base.setdefault(norm, []).append(model_name)
- intended = {_norm_base(d.api_base) for d in self.fleet.deployments if d.api_base}
- now = time.time()
- for dep in self.fleet.deployments:
- base = _norm_base(dep.api_base) if dep.api_base else None
- names = by_base.get(base, []) if base else []
- present = any(
- dep.vllm is not None and dep.vllm.same_deployment(host, port, model_name)
- for _norm, host, port, model_name in routes
- )
- dep.in_litellm = bool(names) or present
- dep.config_mismatch = any(name != dep.model_name for name in names)
- self._store_skew(dep, now)
- extra: list[ExtraLiteLLM] = []
- for norm, host, port, model_name in routes:
- if norm in intended:
- continue
- view = deployment_status(
- DeploymentSnapshot(model_name=model_name, self_hosted=False, litellm_only=True),
- self._cfg.fleet,
- now,
- )
- extra.append(ExtraLiteLLM(host=host, port=port, model_name=model_name, skew=view.skew))
- self.fleet.extra_litellm = extra
- if extra:
- _LOGGER.warning(
- "skew: %s %s",
- SKEW_PHRASE[SKEW_IN_LITELLM_NOT_IN_CONFIG],
- [(row.host, row.port, row.model_name) for row in extra],
- )
-
- async def check_health_skew(self) -> None:
- resp = await self._request("GET", "health")
- self._raise_http(resp)
- body = resp.json() if resp.content else {}
- healthy = {
- _norm_base(x.get("api_base"))
- for x in (body.get("healthy_endpoints") or [])
- if isinstance(x, dict) and x.get("api_base")
- }
- unhealthy = {
- _norm_base(x.get("api_base"))
- for x in (body.get("unhealthy_endpoints") or [])
- if isinstance(x, dict) and x.get("api_base")
- }
- stamp = time.time()
- for dep in self.fleet.deployments:
- base = _norm_base(dep.api_base) if dep.api_base else None
- litellm_up = bool(base) and base in healthy
- litellm_down = bool(base) and base in unhealthy
- if litellm_up:
- dep.litellm_healthy = True
- dep.litellm_health_at = stamp
- elif litellm_down:
- dep.litellm_healthy = False
- dep.litellm_health_at = stamp
- else:
- dep.litellm_healthy = None
- self._store_skew(dep, stamp)
-
- def _store_skew(self, dep, now: float) -> None:
- """Replace stored notes with the badges model_status computes now."""
- view = deployment_status(snapshot_for_status(dep, self._cfg.fleet, now), self._cfg.fleet, now)
- fresh = [badge for badge in view.skew if badge not in dep.skew]
- dep.skew = list(view.skew)
- for badge in fresh:
- _LOGGER.warning("skew: %s@%s %s", dep.name, dep.api_base, SKEW_PHRASE[badge])
-
-
-def _norm_base(url: Any) -> str:
- """Strip trailing slashes and a trailing /v1.
-
- The identity the skew check compares on: the same route appears with the
- suffix (VllmServer.api_base, and the litellm_params.api_base copied from
- it) and without (asf_api_base, routes pushed before the suffix). The
- comparison must match across both forms.
-
- (The suffix was once actively wrong: an older LiteLLM appended
- /v1/chat/completions to api_base, so a /v1-suffixed base resolved to
- /v1/v1/..., 404'd at the origin, and surfaced as an OpenAI
- authentication error naming platform.openai.com -- which sends the reader
- after their own key rather than the route. The current LiteLLM appends
- only /chat/completions, so the suffix is what vLLM needs now.)
- """
- s = str(url or "").rstrip("/")
- if s.endswith("/v1"):
- s = s[:-3].rstrip("/")
- return s
-
-
-def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str]]:
- """Each LiteLLM row as (normalised api_base, host, port, model_name).
-
- Host and port stay separate. One machine serves several ports, and one
- host:port can still report more than one model name.
- """
- rows = []
- if isinstance(body, list):
- rows = body
- elif isinstance(body, dict):
- rows = body.get("data") or body.get("models") or []
- out: list[tuple[str, str, int | None, str]] = []
- for row in rows:
- if not isinstance(row, dict):
- continue
- params = row.get("litellm_params") or {}
- if not isinstance(params, dict) or not params.get("api_base"):
- continue
- norm = _norm_base(params["api_base"])
- parsed = urlparse(norm)
- out.append((norm, parsed.hostname or "", parsed.port, str(row.get("model_name") or "")))
- return out
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
new file mode 100644
index 0000000..832ae71
--- /dev/null
+++ b/llmao/proxy/deployments.py
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""LiteLLM's deployments of the fleet: adding, removing, noticing disagreement.
+
+A deployment is one place a model is served from. LiteLLM sends requests to
+the ones it has been told about, so each has to be registered there when it
+comes up and removed when it goes down, and the two lists compared now and
+then in case they have drifted.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any
+from urllib.parse import urlparse
+
+from llmao.fleet import LEGACY_SERVERS, ExtraLiteLLM, normalize_peer_ip, snapshot_for_status
+from llmao.model_status import (
+ HEALTHY,
+ SKEW_IN_LITELLM_NOT_IN_CONFIG,
+ SKEW_PHRASE,
+ STALLED,
+ UNHEALTHY,
+ DeploymentSnapshot,
+ deployment_status,
+)
+from llmao.proxy.errors import BackendUnavailableError
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class DeploymentManager:
+ """Keeps LiteLLM's deployments in step with the fleet.
+
+ Handed the transport to LiteLLM and the fleet. It holds no state of its
+ own: what it learns is written onto the fleet's FleetDeployment records.
+ """
+
+ def __init__(self, cfg: Any, fleet: Any, transport: Any):
+ self._cfg = cfg
+ self.fleet = fleet
+ self._transport = transport
+
+ def deployment_body(self, dep, *, api_key: str | None = None) -> dict:
+ """POST /model/new payload from models.yaml + this deployment's api_base.
+
+ `api_key` overrides the deployment's own. Callers that can probe the
+ box pass what it actually accepts; see _working_api_key.
+ """
+ entry = self.fleet.models.get(dep.model_name)
+ if entry is None:
+ raise BackendUnavailableError(f"models.yaml missing {dep.model_name}; cannot POST /model/new")
+ params = dict(entry.litellm_params)
+ if not dep.api_base:
+ raise BackendUnavailableError(f"{dep.name}: no api_base for /model/new")
+ params["api_base"] = dep.api_base
+ if dep.self_hosted and dep.vllm is not None and dep.vllm.api_key:
+ params["api_key"] = api_key or dep.vllm.api_key
+
+ info: dict[str, Any] = {
+ "self_hosted": dep.self_hosted,
+ "asf_api_base": _norm_base(dep.api_base),
+ }
+
+ # Price self-hosted tokens so usage reporting shows something other
+ # than 0.00. LiteLLM has no cost-map entry for these models, so every
+ # completion prices at nothing and a project's draw is invisible.
+ #
+ # The price is derived per model in models.yaml from measured
+ # throughput: hourly rental over tokens per second, input and output
+ # separately because prefill runs at thousands per second and decode
+ # at tens. On this fleet that is a 12-27x ratio, close to what
+ # commercial providers charge.
+ #
+ # It follows the card rather than being one fleet-wide figure, because
+ # a faster box costs more per hour and should cost more per token --
+ # otherwise a model's price stops tracking what serving it costs.
+ #
+ # IT IS NOT A MARGINAL COST. The boxes are rented continuously, so a
+ # token costs the same whether or not anyone asks for it, and the
+ # figure above assumes a box decoding flat out. At 25% utilisation the
+ # real cost is four times this; at 5%, twenty times. Read it as a unit
+ # of fair share that happens to carry a dollar sign -- useful for
+ # spotting a runaway job and comparing one project's draw against
+ # another's, and wrong the moment anyone treats it as money the
+ # Foundation could have kept.
+ #
+ # The UI shows it alongside a token count so the distinction survives
+ # contact with a reader.
+ if dep.self_hosted:
+ pricing = getattr(entry, "pricing", None)
+ if pricing is not None:
+ for name in ("input_cost_per_token", "output_cost_per_token"):
+ val = getattr(pricing, name, None)
+ if val is not None:
+ info[name] = float(val)
+ # Marks the row as capacity rather than money, so a report can
+ # keep the two apart instead of adding them together.
+ info["cost_basis"] = dep.cost_basis
+
+ return {
+ "model_name": dep.model_name,
+ "litellm_params": params,
+ "model_info": info,
+ }
+
+ async def _model_info_rows(self) -> list:
+ resp = await self._transport.request("GET", "model/info")
+ self._transport.raise_http(resp)
+ body = resp.json()
+ if isinstance(body, list):
+ return body
+ if isinstance(body, dict):
+ return body.get("data") or body.get("models") or []
+ return []
+
+ def _litellm_id(self, row: dict, dep) -> str | None:
+ if str(row.get("model_name") or "") != dep.model_name:
+ return None
+ want = _norm_base(dep.api_base)
+ info = row.get("model_info") or {}
+ if not isinstance(info, dict):
+ info = {}
+ if (
+ _norm_base(info.get("asf_api_base")) == want
+ or _norm_base(
+ (row.get("litellm_params") or {}).get("api_base") if isinstance(row.get("litellm_params"), dict) else ""
+ )
+ == want
+ ):
+ return str(info.get("id") or row.get("model_id") or "") or None
+ return None
+
+ async def _working_api_key(self, dep) -> str | None:
+ """The bearer to register for this server.
+
+ LEGACY_SERVERS were launched with the shared selfhost_api_key.
+ validate_fleet already required that key when one of those hosts is
+ configured, so this just returns it. Every other box uses the key
+ derived from the salt. No probe: trying the wrong one 401s and the
+ next pass deletes the route.
+ """
+ if not dep.self_hosted or dep.vllm is None:
+ return None
+ srv = dep.vllm
+ if (normalize_peer_ip(srv.host), int(srv.listen_port)) in LEGACY_SERVERS:
+ return str(self._cfg.fleet.selfhost_api_key).strip()
+ return srv.api_key or None
+
+ async def _deployment_present(self, dep) -> bool:
+ if not dep.self_hosted or dep.vllm is None:
+ return False
+ for _norm, host, port, model_name in _routes_from_model_info(await self._model_info_rows()):
+ if dep.vllm is not None and dep.vllm.same_deployment(host, port, model_name):
+ return True
+ return False
+
+ async def add_deployment(self, dep) -> None:
+ """POST /model/new for this deployment. No serving check.
+
+ Callers decide the gate. sync_selfhost waits until vLLM is SERVING.
+ POST /do-add-deployment uses add_refusal for the same gate, then calls
+ this. A deployment already present for this server, on the listen port
+ or the public port, is not posted again. A 400 or 409 means the same thing.
+ """
+ if await self._deployment_present(dep):
+ dep.in_litellm = True
+ _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
+ return
+ body = self.deployment_body(dep, api_key=await self._working_api_key(dep))
+ resp = await self._transport.request("POST", "model/new", json=body)
+ if resp.status_code in (400, 409):
+ dep.in_litellm = True
+ _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
+ return
+ self._transport.raise_http(resp)
+ dep.in_litellm = True
+ _LOGGER.info("model/new %s@%s", dep.name, dep.api_base)
+
+ async def delete_deployment(self, dep) -> None:
+ rows = await self._model_info_rows()
+ found = None
+ for row in rows:
+ if isinstance(row, dict):
+ found = self._litellm_id(row, dep)
+ if found:
+ break
+ if not found:
+ dep.in_litellm = False
+ _LOGGER.warning("model/delete: no LiteLLM id for %s@%s", dep.name, dep.api_base)
+ return
+ resp = await self._transport.request("POST", "model/delete", json={"id": found})
+ self._transport.raise_http(resp)
+ dep.in_litellm = False
+ _LOGGER.info("model/delete %s@%s id=%s", dep.name, dep.api_base, found)
+
+ async def ensure_commercial(self) -> None:
+ """At startup: POST /model/new for each commercial models.yaml deployment."""
+ for dep in self.fleet.deployments:
+ if dep.self_hosted or not dep.api_base:
+ continue
+ if dep.in_litellm:
+ continue
+ await self.add_deployment(dep)
+
+ async def sync_selfhost(self) -> None:
+ """After vLLM probes: register serving backends, drop down ones."""
+ for dep in self.fleet.deployments:
+ if not dep.self_hosted or dep.vllm is None:
+ continue
+ srv = dep.vllm
+ if srv.state == HEALTHY and dep.api_base and not dep.in_litellm:
+ await self.add_deployment(dep)
+ elif srv.state in (UNHEALTHY, STALLED) and dep.in_litellm:
+ await self.delete_deployment(dep)
+
+ async def run_skew(self) -> None:
+ """Config skew often; LiteLLM GET /health (tokens) on a long interval."""
+ skew_s = float(self._cfg.fleet.skew_interval_s)
+ health_s = float(self._cfg.fleet.litellm_health_interval_s)
+ last_health = 0.0
+ first = True
+ while True:
+ try:
+ await self.check_config_skew()
+ now = time.time()
+ if first or (now - last_health) >= health_s:
+ await self.check_health_skew()
+ last_health = now
+ first = False
+ except Exception:
+ _LOGGER.exception("LiteLLM skew check failed")
+ await asyncio.sleep(skew_s)
+
+ async def check_config_skew(self) -> None:
+ resp = await self._transport.request("GET", "model/info")
+ self._transport.raise_http(resp)
+ routes = _routes_from_model_info(resp.json())
+ by_base: dict[str, list[str]] = {}
+ for norm, _host, _port, model_name in routes:
+ by_base.setdefault(norm, []).append(model_name)
+ intended = {_norm_base(d.api_base) for d in self.fleet.deployments if d.api_base}
+ now = time.time()
+ for dep in self.fleet.deployments:
+ base = _norm_base(dep.api_base) if dep.api_base else None
+ names = by_base.get(base, []) if base else []
+ present = any(
+ dep.vllm is not None and dep.vllm.same_deployment(host, port, model_name)
+ for _norm, host, port, model_name in routes
+ )
+ dep.in_litellm = bool(names) or present
+ dep.config_mismatch = any(name != dep.model_name for name in names)
+ self._store_skew(dep, now)
+ extra: list[ExtraLiteLLM] = []
+ for norm, host, port, model_name in routes:
+ if norm in intended:
+ continue
+ view = deployment_status(
+ DeploymentSnapshot(model_name=model_name, self_hosted=False, litellm_only=True),
+ self._cfg.fleet,
+ now,
+ )
+ extra.append(ExtraLiteLLM(host=host, port=port, model_name=model_name, skew=view.skew))
+ self.fleet.extra_litellm = extra
+ if extra:
+ _LOGGER.warning(
+ "skew: %s %s",
+ SKEW_PHRASE[SKEW_IN_LITELLM_NOT_IN_CONFIG],
+ [(row.host, row.port, row.model_name) for row in extra],
+ )
+
+ async def check_health_skew(self) -> None:
+ resp = await self._transport.request("GET", "health")
+ self._transport.raise_http(resp)
+ body = resp.json() if resp.content else {}
+ healthy = {
+ _norm_base(x.get("api_base"))
+ for x in (body.get("healthy_endpoints") or [])
+ if isinstance(x, dict) and x.get("api_base")
+ }
+ unhealthy = {
+ _norm_base(x.get("api_base"))
+ for x in (body.get("unhealthy_endpoints") or [])
+ if isinstance(x, dict) and x.get("api_base")
+ }
+ stamp = time.time()
+ for dep in self.fleet.deployments:
+ base = _norm_base(dep.api_base) if dep.api_base else None
+ litellm_up = bool(base) and base in healthy
+ litellm_down = bool(base) and base in unhealthy
+ if litellm_up:
+ dep.litellm_healthy = True
+ dep.litellm_health_at = stamp
+ elif litellm_down:
+ dep.litellm_healthy = False
+ dep.litellm_health_at = stamp
+ else:
+ dep.litellm_healthy = None
+ self._store_skew(dep, stamp)
+
+ def _store_skew(self, dep, now: float) -> None:
+ """Replace stored notes with the badges model_status computes now."""
+ view = deployment_status(snapshot_for_status(dep, self._cfg.fleet, now), self._cfg.fleet, now)
+ fresh = [badge for badge in view.skew if badge not in dep.skew]
+ dep.skew = list(view.skew)
+ for badge in fresh:
+ _LOGGER.warning("skew: %s@%s %s", dep.name, dep.api_base, SKEW_PHRASE[badge])
+
+
+def _norm_base(url: Any) -> str:
+ """Strip trailing slashes and a trailing /v1.
+
+ The identity the skew check compares on: the same route appears with the
+ suffix (VllmServer.api_base, and the litellm_params.api_base copied from
+ it) and without (asf_api_base, routes pushed before the suffix). The
+ comparison must match across both forms.
+
+ (The suffix was once actively wrong: an older LiteLLM appended
+ /v1/chat/completions to api_base, so a /v1-suffixed base resolved to
+ /v1/v1/..., 404'd at the origin, and surfaced as an OpenAI
+ authentication error naming platform.openai.com -- which sends the reader
+ after their own key rather than the route. The current LiteLLM appends
+ only /chat/completions, so the suffix is what vLLM needs now.)
+ """
+ s = str(url or "").rstrip("/")
+ if s.endswith("/v1"):
+ s = s[:-3].rstrip("/")
+ return s
+
+
+def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str]]:
+ """Each LiteLLM row as (normalised api_base, host, port, model_name).
+
+ Host and port stay separate. One machine serves several ports, and one
+ host:port can still report more than one model name.
+ """
+ rows = []
+ if isinstance(body, list):
+ rows = body
+ elif isinstance(body, dict):
+ rows = body.get("data") or body.get("models") or []
+ out: list[tuple[str, str, int | None, str]] = []
+ for row in rows:
+ if not isinstance(row, dict):
+ continue
+ params = row.get("litellm_params") or {}
+ if not isinstance(params, dict) or not params.get("api_base"):
+ continue
+ norm = _norm_base(params["api_base"])
+ parsed = urlparse(norm)
+ out.append((norm, parsed.hostname or "", parsed.port, str(row.get("model_name") or "")))
+ return out
diff --git a/llmao/proxy/errors.py b/llmao/proxy/errors.py
new file mode 100644
index 0000000..d7885b4
--- /dev/null
+++ b/llmao/proxy/errors.py
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""What can go wrong talking to LiteLLM, as the pages see it.
+
+Here rather than beside the code that raises them because that code is in
+more than one module, and each would otherwise have to import another for
+the sake of an exception.
+"""
+
+
+class BudgetExceededError(Exception):
+ """Raised when a team is over budget (mirrors litellm proxy's 4xx)."""
+
+
+class BackendUnavailableError(Exception):
+ """Raised when the litellm admin API times out or can't be reached."""
diff --git a/main.py b/main.py
index e42c9ca..44a2076 100644
--- a/main.py
+++ b/main.py
@@ -81,10 +81,10 @@ def create_app():
fleet = Fleet.from_cfg(app.cfg, models=definitions)
backend = LiteLLMBackend(app.cfg, fleet)
- fleet.after_probe = backend.sync_selfhost
+ fleet.after_probe = backend.deployments.sync_selfhost
app.fleet = fleet
app.add_runner(fleet.run_lifecycle, name="fleet-lifecycle")
- app.add_runner(backend.run_skew, name="litellm-skew")
+ app.add_runner(backend.deployments.run_skew, name="litellm-skew")
# Sample queue depth on each box.
#
diff --git a/pages.py b/pages.py
index d056a2e..69fb1a4 100644
--- a/pages.py
+++ b/pages.py
@@ -914,7 +914,7 @@ async def do_add_deployment():
await flash_danger(reason)
return _see_other("/models")
try:
- await APP.backend.add_deployment(matches[0])
+ await APP.backend.deployments.add_deployment(matches[0])
except BackendUnavailableError as e:
await flash_danger(str(e))
return _see_other("/models")
@@ -944,7 +944,7 @@ async def do_remove_deployment():
await flash_danger(reason)
return _see_other("/models")
try:
- await APP.backend.delete_deployment(matches[0])
+ await APP.backend.deployments.delete_deployment(matches[0])
except BackendUnavailableError as e:
await flash_danger(str(e))
return _see_other("/models")
diff --git a/tests/test_access_requests.py b/tests/test_access_requests.py
index c50fdba..efe31bf 100644
--- a/tests/test_access_requests.py
+++ b/tests/test_access_requests.py
@@ -27,7 +27,6 @@ import json
import pytest
from easydict import EasyDict as edict # noqa: N813
-from fake_transport import FakeTransport
from llmao import access_requests
from llmao.access_requests import (
@@ -45,6 +44,7 @@ from llmao.access_requests import (
prune,
)
from llmao.proxy.client import LiteLLMBackend
+from tests.fake_transport import FakeTransport
UTC = dt.UTC
diff --git a/tests/test_fleet.py b/tests/test_fleet.py
index b30522a..7b0804e 100644
--- a/tests/test_fleet.py
+++ b/tests/test_fleet.py
@@ -38,7 +38,7 @@ from llmao.fleet import (
)
from llmao.model_status import AWAITING, HEALTHY, LOADING, deployment_status
from llmao.models import load_models
-from llmao.proxy.client import _norm_base
+from llmao.proxy.deployments import _norm_base
from llmao.vllm_api_key import derive_vllm_api_key
EXAMPLE = Path(__file__).resolve().parent.parent / "models.yaml"
@@ -391,9 +391,9 @@ def test_working_api_key_follows_the_legacy_set():
in_range = edict(self_hosted=True, vllm=edict(host="213.173.110.78", listen_port=8003, api_key="sk-derived"))
async def run():
- assert await backend._working_api_key(legacy) == "sk-shared"
- assert await backend._working_api_key(in_range) == "sk-shared"
- assert await backend._working_api_key(other) == "sk-derived"
+ assert await backend.deployments._working_api_key(legacy) == "sk-shared"
+ assert await backend.deployments._working_api_key(in_range) == "sk-shared"
+ assert await backend.deployments._working_api_key(other) == "sk-derived"
await backend.aclose()
asyncio.run(run())
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index 898c7bb..4956fc2 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -45,7 +45,8 @@ from llmao.model_status import (
UNAVAILABLE,
UNHEALTHY,
)
-from llmao.proxy.client import LiteLLMBackend
+from llmao.proxy.deployments import DeploymentManager
+from tests.fake_transport import FakeTransport
from tests.test_fleet import FLEET_KNOBS
@@ -154,25 +155,21 @@ def test_add_refusal_until_serving():
assert add_refusal(dep) == "already in LiteLLM"
-class _Backend:
- def __init__(self, fleet, info, health):
- self.fleet = fleet
+class _LiteLLM:
+ """What the stand-in LiteLLM says. A test changes these as it goes."""
+
+ def __init__(self, info, health):
self.info = info
self.health = health
- self.added = []
- async def _request(self, method, path, **kwargs):
- body = self.info if path == "model/info" else self.health
- resp = _Resp(body=body)
- resp.content = b"{}"
- return resp
+ def __call__(self, call):
+ return self.info if call.path == "model/info" else self.health
- def _raise_http(self, resp):
- return None
- async def add_deployment(self, dep):
- self.added.append(dep)
- dep.in_litellm = True
+def _skew_manager(fleet, info, health=None):
+ """A DeploymentManager, and the stand-in LiteLLM it is talking to."""
+ litellm = _LiteLLM(info, health or {})
+ return DeploymentManager(fleet.cfg, fleet, FakeTransport(litellm)), litellm
def test_skew_sentences_replace_old_phrases():
@@ -187,37 +184,28 @@ def test_skew_sentences_replace_old_phrases():
dep = FleetDeployment.from_vllm(srv)
dep.skew = ["missing from LiteLLM", "LiteLLM health disagrees"]
fleet = Fleet(cfg=edict({"fleet": {"hosts": {}, **FLEET_KNOBS}}), servers=[srv], deployments=[dep])
- backend = _Backend(fleet, info=[], health={})
- backend._cfg = fleet.cfg
- backend._store_skew = lambda dep, now: LiteLLMBackend._store_skew(backend, dep, now)
- asyncio.run(LiteLLMBackend.check_config_skew(backend))
+ backend, litellm = _skew_manager(fleet, [])
+ asyncio.run(backend.check_config_skew())
# Loading is not a missing-route badge. The old sentence is dropped.
assert dep.skew == []
assert "missing from LiteLLM" not in dep.skew
srv.record_probe(True, now=time.time(), grace_s=1800, fail_threshold=3)
assert srv.state == HEALTHY
- asyncio.run(LiteLLMBackend.check_config_skew(backend))
+ asyncio.run(backend.check_config_skew())
assert dep.skew == [SKEW_INTENDED_NOT_IN_LITELLM]
base = srv.api_base
- backend.health = {"unhealthy_endpoints": [{"api_base": base}], "healthy_endpoints": []}
- asyncio.run(LiteLLMBackend.check_health_skew(backend))
+ litellm.health = {"unhealthy_endpoints": [{"api_base": base}], "healthy_endpoints": []}
+ asyncio.run(backend.check_health_skew())
assert SKEW_VLLM_UP_LITELLM_DOWN in dep.skew
for _ in range(3):
srv.record_probe(False, now=time.time(), grace_s=1800, fail_threshold=3, err="refused")
assert srv.state == UNHEALTHY
- backend.health = {"healthy_endpoints": [{"api_base": base}], "unhealthy_endpoints": []}
- asyncio.run(LiteLLMBackend.check_health_skew(backend))
+ litellm.health = {"healthy_endpoints": [{"api_base": base}], "unhealthy_endpoints": []}
+ asyncio.run(backend.check_health_skew())
assert SKEW_VLLM_DOWN_LITELLM_UP in dep.skew
assert "LiteLLM health disagrees" not in dep.skew
-def _skew_backend(fleet, info):
- backend = _Backend(fleet, info=info, health={})
- backend._cfg = fleet.cfg
- backend._store_skew = lambda dep, now: LiteLLMBackend._store_skew(backend, dep, now)
- return backend
-
-
def test_extra_routes_are_per_port_and_model():
srv = _server()
dep = FleetDeployment.from_vllm(srv)
@@ -227,7 +215,7 @@ def test_extra_routes_are_per_port_and_model():
{"model_name": "other-b", "litellm_params": {"api_base": "http://10.0.0.1:9002/v1"}},
{"model_name": "other-c", "litellm_params": {"api_base": "http://10.0.0.1:9002/v1"}},
]
- asyncio.run(LiteLLMBackend.check_config_skew(_skew_backend(fleet, info)))
+ asyncio.run(_skew_manager(fleet, info)[0].check_config_skew())
found = {(row.host, row.port, row.model_name) for row in fleet.extra_litellm}
assert found == {("10.0.0.1", 9001, "other-a"), ("10.0.0.1", 9002, "other-b"), ("10.0.0.1", 9002, "other-c")}
assert all(SKEW_IN_LITELLM_NOT_IN_CONFIG in row.skew for row in fleet.extra_litellm)
@@ -239,14 +227,14 @@ def test_model_name_mismatch_survives_health_check():
srv.state = HEALTHY
dep = FleetDeployment.from_vllm(srv)
fleet = Fleet(cfg=edict({"fleet": {"hosts": {}, **FLEET_KNOBS}}), servers=[srv], deployments=[dep])
- backend = _skew_backend(fleet, [{"model_name": "not-gemma", "litellm_params": {"api_base": srv.api_base}}])
- asyncio.run(LiteLLMBackend.check_config_skew(backend))
+ backend, litellm = _skew_manager(fleet, [{"model_name": "not-gemma", "litellm_params": {"api_base": srv.api_base}}])
+ asyncio.run(backend.check_config_skew())
assert SKEW_CONFIG_MISMATCH in dep.skew
- backend.health = {"healthy_endpoints": [{"api_base": srv.api_base}], "unhealthy_endpoints": []}
- asyncio.run(LiteLLMBackend.check_health_skew(backend))
+ litellm.health = {"healthy_endpoints": [{"api_base": srv.api_base}], "unhealthy_endpoints": []}
+ asyncio.run(backend.check_health_skew())
assert SKEW_CONFIG_MISMATCH in dep.skew
- backend.info = [{"model_name": srv.model_name, "litellm_params": {"api_base": srv.api_base}}]
- asyncio.run(LiteLLMBackend.check_config_skew(backend))
+ litellm.info = [{"model_name": srv.model_name, "litellm_params": {"api_base": srv.api_base}}]
+ asyncio.run(backend.check_config_skew())
assert SKEW_CONFIG_MISMATCH not in dep.skew
assert fleet.extra_litellm == []
@@ -255,19 +243,19 @@ def test_add_calls_deployment_only_when_serving():
srv = _server()
dep = FleetDeployment.from_vllm(srv)
fleet = Fleet(cfg=edict({"fleet": {"hosts": {}}}), servers=[srv], deployments=[dep])
- backend = _Backend(fleet, info=[], health={})
+ added = []
- async def attempt():
+ def attempt():
+ """What the Add button does: ask add_refusal, and add only if it does not object."""
if add_refusal(dep):
return
- await backend.add_deployment(dep)
+ added.append(dep)
- asyncio.run(attempt())
- assert backend.added == []
+ attempt()
+ assert added == []
srv.state = HEALTHY
- asyncio.run(attempt())
- assert backend.added == [dep]
- assert dep.in_litellm is True
+ attempt()
+ assert added == [dep]
def test_box_json_listen_not_public():
diff --git a/tests/test_mix.py b/tests/test_mix.py
index 533a1a7..324b56e 100644
--- a/tests/test_mix.py
+++ b/tests/test_mix.py
@@ -23,20 +23,19 @@ from easydict import EasyDict as edict # noqa: N813
from llmao.fleet import Fleet, FleetDeployment
from llmao.model_status import HEALTHY, UNHEALTHY
-from llmao.proxy.client import LiteLLMBackend
+from llmao.proxy.deployments import DeploymentManager
+from tests.fake_transport import FakeTransport
from tests.test_fleet_health import _server
-def _backend(fleet):
- cfg = edict(
- litellm=edict(
- base_url="http://127.0.0.1:9",
- master_key="sk-x",
- request_timeout_s=1,
- ),
- fleet=edict(),
- )
- return LiteLLMBackend(cfg, fleet)
+def _manager(fleet, answer=None):
+ """A DeploymentManager over a stand-in LiteLLM that answers every call with `answer`."""
+ cfg = edict(fleet=edict())
+ return DeploymentManager(cfg, fleet, FakeTransport(lambda call: answer))
+
+
+def _paths(manager):
+ return [c.path for c in manager._transport.calls]
def test_deployment_body_from_models():
@@ -45,7 +44,7 @@ def test_deployment_body_from_models():
"gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b")),
}
fleet = Fleet(None, [s], models=models)
- body = _backend(fleet).deployment_body(fleet.deployments[0])
+ body = _manager(fleet).deployment_body(fleet.deployments[0])
assert body["model_name"] == "gemma4-26b"
assert body["litellm_params"]["model"] == "hosted_vllm/gemma4-26b"
# api_base keeps the /v1 LiteLLM needs (it appends /chat/completions to it);
@@ -63,27 +62,10 @@ def test_sync_selfhost_posts_new_when_serving():
"gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b")),
}
fleet = Fleet(None, [s], models=models)
- be = _backend(fleet)
- calls = []
-
- async def _req(method, path, **kw):
- calls.append((method, path, kw.get("json")))
-
- class R:
- status_code = 200
- content = b"{}"
-
- def raise_for_status(self):
- return None
-
- def json(self):
- return {"data": []}
-
- return R()
-
- be._request = _req
+ be = _manager(fleet, {"data": []})
asyncio.run(be.sync_selfhost())
- assert ("POST", "model/new") == (calls[-1][0], calls[-1][1])
+ last = be._transport.calls[-1]
+ assert (last.method, last.path) == ("POST", "model/new")
assert fleet.deployments[0].in_litellm is True
@@ -95,21 +77,7 @@ def test_sync_selfhost_deletes_when_down():
}
fleet = Fleet(None, [s], models=models)
fleet.deployments[0].in_litellm = True
- be = _backend(fleet)
-
- async def _req(method, path, **kw):
- class R:
- status_code = 200
-
- def raise_for_status(self):
- return None
-
- def json(self):
- return _info("10.0.0.1", 8001, model_id="abc")
-
- return R()
-
- be._request = _req
+ be = _manager(fleet, _info("10.0.0.1", 8001, model_id="abc"))
asyncio.run(be.sync_selfhost())
assert fleet.deployments[0].in_litellm is False
@@ -132,27 +100,9 @@ def test_sync_selfhost_does_not_repost_the_same_deployment():
for port in (12711, 10100):
fleet = Fleet(None, [s], models=models)
fleet.deployments[0].in_litellm = False
- be = _backend(fleet)
- calls = []
-
- async def _req(method, path, _calls=calls, _port=port, **kw):
- _calls.append(path)
-
- class R:
- status_code = 200
- content = b"{}"
-
- def raise_for_status(self):
- return None
-
- def json(self):
- return _info("80.188.223.202", _port)
-
- return R()
-
- be._request = _req
+ be = _manager(fleet, _info("80.188.223.202", port))
asyncio.run(be.sync_selfhost())
- assert "model/new" not in calls
+ assert "model/new" not in _paths(be)
assert fleet.deployments[0].in_litellm is True
@@ -161,27 +111,9 @@ def test_other_port_is_not_this_deployment():
s.state = HEALTHY
models = {"gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b"))}
fleet = Fleet(None, [s], models=models)
- be = _backend(fleet)
- calls = []
-
- async def _req(method, path, **kw):
- calls.append(path)
-
- class R:
- status_code = 200
- content = b"{}"
-
- def raise_for_status(self):
- return None
-
- def json(self):
- return _info("10.1.2.3", 19999)
-
- return R()
-
- be._request = _req
+ be = _manager(fleet, _info("10.1.2.3", 19999))
asyncio.run(be.sync_selfhost())
- assert "model/new" in calls
+ assert "model/new" in _paths(be)
def test_ensure_commercial():
@@ -198,25 +130,7 @@ def test_ensure_commercial():
)
}
fleet = Fleet(None, [], deployments=[dep], models=models)
- be = _backend(fleet)
- calls = []
-
- async def _req(method, path, **kw):
- calls.append(path)
-
- class R:
- status_code = 200
- content = b"{}"
-
- def raise_for_status(self):
- return None
-
- def json(self):
- return {"data": []}
-
- return R()
-
- be._request = _req
+ be = _manager(fleet, {"data": []})
asyncio.run(be.ensure_commercial())
- assert calls == ["model/new"]
+ assert _paths(be) == ["model/new"]
assert dep.in_litellm is True
diff --git a/tests/test_spend_rows.py b/tests/test_spend_rows.py
index 175a0cb..00b0608 100644
--- a/tests/test_spend_rows.py
+++ b/tests/test_spend_rows.py
@@ -25,10 +25,10 @@ import asyncio
import datetime as dt
from easydict import EasyDict as edict # noqa: N813
-from fake_transport import FakeTransport
from llmao.fleet import Fleet, FleetDeployment
from llmao.proxy.client import LiteLLMBackend
+from tests.fake_transport import FakeTransport
NOON = dt.datetime(2026, 10, 12, 12, 0, tzinfo=dt.UTC)
TODAY = "2026-10-12"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-llmao) 01/03: fleet: a deployment knows whether its cost is capacity or money
Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 62c5b33ff30808785a917f1eec93af2dd38c5056
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Mon Oct 12 18:30:35 2026 -0500
fleet: a deployment knows whether its cost is capacity or money
The backend kept a private dict, model name to cost basis, filled as a side
effect of building a route definition for LiteLLM. That only happens when a
route is added. After a restart with the routes already in place nothing is
added, the dict stays empty, and every usage row reads as real money.
It was never a fact about API calls. Where a deployment runs decides it: on
our own boxes the dollars are derived from throughput, and from a provider
they are a bill. So FleetDeployment says which, and the usage rows ask the
fleet by model name.
A name served both ways, or by nothing, is treated as money. Usage is
recorded per name, the two cannot be told apart, and calling the lot an
estimate would hide a bill.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
llmao/fleet.py | 28 ++++++++++++++++++++++++++++
llmao/litellm_client.py | 19 +++++++------------
tests/test_spend_rows.py | 27 +++++++++++++++++++++++++--
3 files changed, 60 insertions(+), 14 deletions(-)
diff --git a/llmao/fleet.py b/llmao/fleet.py
index b8eaf77..913d92e 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -533,6 +533,10 @@ class VllmServer:
_LOGGER.warning("fleet server %s@%s %s (%s)", self.name, self.api_base, self.state, self.last_error)
+# What a deployment's cost figure means, when it is not simply money owed.
+COST_BASIS_CAPACITY = "derived_capacity"
+
+
class FleetDeployment:
"""One intended LiteLLM deployment (definitions + api_base).
@@ -587,6 +591,18 @@ class FleetDeployment:
return self.vllm.api_base
return self._api_base
+ @property
+ def cost_basis(self) -> str | None:
+ """Whether this deployment's cost is a capacity estimate or real money.
+
+ A fact about where the deployment runs, and nothing else. The same
+ model can be served from our own boxes or bought from a provider:
+ on our boxes the dollars are derived from throughput and are a share
+ of capacity already paid for, and from a provider they are a bill.
+ None means real money.
+ """
+ return COST_BASIS_CAPACITY if self.self_hosted else None
+
@property
def litellm_health_display(self) -> str:
"""Human-readable 'litellm_health_at'; a dash when unknown."""
@@ -882,6 +898,18 @@ class Fleet:
catalog = bool(flag)
return model_status(model_name, views, catalog_self_hosted=catalog)
+ def cost_basis(self, model_name: str) -> str | None:
+ """How to read a cost recorded against this model name.
+
+ Usage is recorded per model name, not per deployment, so this is as
+ fine as it can be told apart. A capacity estimate only when every
+ deployment of the name is one: where a name is served both ways,
+ some of its dollars are real, and calling the lot an estimate would
+ hide a bill. A name with no deployment is likewise treated as money.
+ """
+ bases = {d.cost_basis for d in self.deployments if d.model_name == model_name}
+ return COST_BASIS_CAPACITY if bases == {COST_BASIS_CAPACITY} else None
+
def model_in_litellm(self, model_name: str) -> bool:
"""True if any vLLM for this model_name has a LiteLLM deployment (skew)."""
return any(d.model_name == model_name and d.in_litellm for d in self.deployments)
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index e5eac4e..5259480 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -399,9 +399,6 @@ class LiteLLMBackend:
self._daily_span = 0
# (monotonic stamp, team/list rows).
self._team_rows_cache: tuple[float, list[dict[str, Any]]] | None = None
- # model_name -> cost_basis, filled when routes are registered, so a
- # spend row can be told from an invoice without a second lookup.
- self._cost_basis: dict[str, str] = {}
async def aclose(self) -> None:
await self._transport.aclose()
@@ -1068,8 +1065,9 @@ class LiteLLMBackend:
three queries would be three chances to disagree.
`cost_basis` is carried through so a caller can tell an estimate from
- an amount payable. Self-hosted routes are tagged derived_capacity by
- deployment_body; commercial rows have no tag and their cost is real.
+ an amount payable. It comes from the fleet: a model served from our
+ own boxes is derived_capacity; anything else has no tag and its cost
+ is real.
"""
rows = await self._spend_rows()
if project is None:
@@ -1121,7 +1119,7 @@ class LiteLLMBackend:
"token_id": row["token_id"],
"total_tokens": 0,
"total_cost_usd": 0.0,
- "cost_basis": self._cost_basis.get(row["model"]),
+ "cost_basis": self.fleet.cost_basis(row["model"]) if self.fleet else None,
"requests": 0,
}
acc["total_tokens"] += row["total_tokens"]
@@ -1176,8 +1174,8 @@ class LiteLLMBackend:
"""One read of LiteLLM's daily aggregate, as rows grouped by date.
The answer nests day -> model group -> key. The model GROUP is the
- route name llmao registered, which is what cost_basis is keyed on;
- the `models` breakdown beside it names the provider's model instead.
+ route name llmao registered, which is the name the fleet knows it
+ by; the `models` breakdown beside it names the provider's model.
"""
out: dict[str, list[dict]] = {}
page = 1
@@ -1347,10 +1345,7 @@ class LiteLLMBackend:
info[name] = float(val)
# Marks the row as capacity rather than money, so a report can
# keep the two apart instead of adding them together.
- info["cost_basis"] = "derived_capacity"
- # Remembered so a spend row can be classified without a
- # second call to /model/info per page load.
- self._cost_basis[dep.model_name] = "derived_capacity"
+ info["cost_basis"] = dep.cost_basis
return {
"model_name": dep.model_name,
diff --git a/tests/test_spend_rows.py b/tests/test_spend_rows.py
index 8c4562d..fc94223 100644
--- a/tests/test_spend_rows.py
+++ b/tests/test_spend_rows.py
@@ -27,6 +27,7 @@ import datetime as dt
from easydict import EasyDict as edict # noqa: N813
from fake_transport import FakeTransport
+from llmao.fleet import Fleet, FleetDeployment
from llmao.litellm_client import LiteLLMBackend
NOON = dt.datetime(2026, 10, 12, 12, 0, tzinfo=dt.UTC)
@@ -77,9 +78,17 @@ def _backend(days, *, now=NOON, page_size=1000):
"metadata": {"has_more": first + p.page_size < len(hits)},
}
- b = LiteLLMBackend(edict(), None, transport=FakeTransport(answer))
+ # gemma runs on our own box; claude-opus-5 is bought in
+ fleet = Fleet(
+ None,
+ [],
+ deployments=[
+ FleetDeployment(model_name="gemma4-26b", name="gemma4-26b", self_hosted=True),
+ FleetDeployment(model_name="claude-opus-5", name="claude-opus-5", self_hosted=False),
+ ],
+ )
+ b = LiteLLMBackend(edict(), fleet, transport=FakeTransport(answer))
b._team_ids = dict(TEAMS)
- b._cost_basis = {"gemma4-26b": "derived_capacity"}
b._DAILY_PAGE_SIZE = page_size
b._now = lambda: now
return b
@@ -185,6 +194,20 @@ def test_cost_basis_distinguishes_capacity_from_money():
assert real_money == 1.25
+def test_a_model_served_both_ways_is_not_called_an_estimate():
+ """The same name can be on our boxes and bought in.
+
+ Usage is recorded per name, so the two cannot be told apart, and some of
+ those dollars are a bill. Calling the lot an estimate would hide it.
+ """
+ ours = FleetDeployment(model_name="m", name="m", self_hosted=True)
+ bought = FleetDeployment(model_name="m", name="m-bedrock", self_hosted=False)
+ assert Fleet(None, [], deployments=[ours]).cost_basis("m") == "derived_capacity"
+ assert Fleet(None, [], deployments=[ours, bought]).cost_basis("m") is None
+ assert Fleet(None, [], deployments=[bought]).cost_basis("m") is None
+ assert Fleet(None, [], deployments=[]).cost_basis("never-heard-of-it") is None
+
+
def test_a_key_llmao_did_not_issue_is_kept():
"""No owner, no team -- but it still consumed the fleet.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org