You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/08/11 12:27:37 UTC
(tooling-llmao) branch main updated: In-process team_id cache with fail-fast warm; drop StateStore.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
The following commit(s) were added to refs/heads/main by this push:
new 5be037b In-process team_id cache with fail-fast warm; drop StateStore.
5be037b is described below
commit 5be037b0db584369241bc61e085837debf576b51
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Tue Aug 11 07:27:30 2026 -0500
In-process team_id cache with fail-fast warm; drop StateStore.
Cache only project→team_id (immutable pair), fill at before_serving via
team/list. team_info uses team/info on cache hit or list-row data on miss.
Instantiate LiteLLMBackend directly; remove disk state store and make_backend.
Co-authored-by: Grok <gr...@x.ai>
---
README.md | 2 +-
config.yaml.example | 1 -
docs/ARCHITECTURE.md | 8 ++-
docs/STATUS.md | 2 +
llmao/litellm_client.py | 160 +++++++++++++++++++++++++++---------------------
llmao/store.py | 44 -------------
main.py | 15 +++--
7 files changed, 112 insertions(+), 120 deletions(-)
diff --git a/README.md b/README.md
index ebf270e..0c62640 100644
--- a/README.md
+++ b/README.md
@@ -205,6 +205,6 @@ config.yaml.example → config.yaml (gitignored)
litellm.yaml.example → litellm.yaml (include model_list.yaml)
model_list.yaml.example → model_list.yaml (inventory SoT; keys from eyaml)
certs/ mkcert PEMs + README
-llmao/ seam, auth, models, litellm_client, store
+llmao/ seam, auth, models, litellm_client
tests/ offline seam + model_list / LiteLLM metadata tests
```
diff --git a/config.yaml.example b/config.yaml.example
index d70bb4a..72fe35c 100644
--- a/config.yaml.example
+++ b/config.yaml.example
@@ -36,7 +36,6 @@ budgets:
# uids that may view any project's activity (plus asfquart isRoot)
site_admins: []
-state_path: ./llmao-state.json
# Shared model inventory (LiteLLM include + llmao UX). Relative to app root.
models_path: model_list.yaml
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 7f1bc32..6dfae43 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -61,12 +61,18 @@ The app **always** uses **LiteLLMBackend** against a real LiteLLM admin API.
Offline **MockBackend** lives under `tests/` and is injected only by unit
tests—not a second runtime mode.
+**Team ids:** in-process cache `project (team_alias) → team_id` only (immutable
+under our rules). Warmed at startup (`before_serving` + `warm()`, fail-fast if
+LiteLLM is down). Spend/budget always from live `team/info` when the id is
+known; after a cache-miss `team/list`, use list-row fields (no redundant
+`team/info`). No on-disk state store.
+
## The seam
`seam.py`:
1. **authorizes** project membership / PMC admin after asfquart has authenticated;
-2. **resolves** the ASF project to a LiteLLM team (budget on first use).
+2. **delegates** project-scoped team/key ops to the backend (product API speaks project).
## Request path (this process)
diff --git a/docs/STATUS.md b/docs/STATUS.md
index c818476..30e10e6 100644
--- a/docs/STATUS.md
+++ b/docs/STATUS.md
@@ -33,7 +33,9 @@ Detailed “where we are.” Top-level README stays short and links here.
- Design names on `KeyInfo`: `project`, `user`, `purpose`; `token_id` for list/revoke (not the secret). Automation ⇔ `user is None` (no separate `kind`).
- Automation keys store **`metadata.created_by`** (ASF uid who minted and saw the secret); surfaced as `KeyInfo.created_by` so other PMCs know who to ask.
- Seam/Backend product API speaks **project** (LDAP); LiteLLM `team_id` is resolved only inside `LiteLLMBackend`.
+- In-process **project→team_id** cache only (immutable pair); warmed at startup via `before_serving` + `LiteLLMBackend.warm()` (**fail-fast** if LiteLLM is down). Spend always from live `team/info` when id is known.
- Wire map: purpose↔`key_alias`, user↔`user_id`, project↔`metadata.project` + team_alias, token_id↔LiteLLM `token`, created_by↔`metadata.created_by`.
+- No on-disk `StateStore` / `state_path`.
- Secret `sk-…` shown **once** (`CreatedKey.secret`); not kept on `KeyInfo`.
### Open policy: who may create automation PATs?
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index 486a33d..4c0d02f 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -8,14 +8,11 @@ LiteLLM opaque team_id is internal to LiteLLMBackend only.
"""
from __future__ import annotations
-import time
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Protocol
import httpx
-from .store import StateStore
-
class BudgetExceeded(Exception):
"""Raised when a team is over budget (mirrors litellm proxy's 4xx)."""
@@ -155,11 +152,15 @@ def _normalize_key_obj(raw: Any) -> KeyInfo:
class LiteLLMBackend:
- """Talks to a running litellm proxy via httpx.AsyncClient."""
+ """Talks to a running litellm proxy via httpx.AsyncClient.
+
+ In-process cache is only project (team_alias) → team_id. Spend/budget are
+ never taken from the cache. Call ``warm()`` at process start (fail-fast).
+ """
- def __init__(self, cfg: Any, store: StateStore):
+ def __init__(self, cfg: Any):
self._cfg = cfg
- self._store = store
+ self._team_ids: Dict[str, str] = {} # project → team_id
base = cfg.litellm.base_url.rstrip("/") + "/"
timeout_s = int(cfg.litellm.request_timeout_s)
self._client = httpx.AsyncClient(
@@ -195,43 +196,60 @@ class LiteLLMBackend:
f"LiteLLM admin error {e.response.status_code}: {e.response.text}"
) from e
- def _remember_team(self, project: str, team_id: str, budget_usd: float) -> None:
- def _mut(data):
- data.setdefault("teams", {})[project] = {
- "team_id": team_id,
- "max_budget": budget_usd,
- "spend": 0.0,
- "created_at": time.time(),
- }
- self._store.update(_mut)
-
- async def _find_team_id_by_alias(self, project: str) -> Optional[str]:
+ def _ingest_team_list(self, teams: List[Any]) -> List[Dict[str, Any]]:
+ """Update project→team_id cache from team/list rows; return dict rows."""
+ rows: List[Dict[str, Any]] = []
+ for t in teams:
+ if hasattr(t, "model_dump"):
+ d = t.model_dump()
+ elif hasattr(t, "dict"):
+ d = t.dict()
+ elif isinstance(t, dict):
+ d = t
+ else:
+ continue
+ rows.append(d)
+ alias = d.get("team_alias")
+ tid = d.get("team_id")
+ if alias and tid:
+ self._team_ids[str(alias)] = str(tid)
+ return rows
+
+ async def _team_list_rows(self) -> List[Dict[str, Any]]:
resp = await self._request("GET", "team/list")
- if not resp.is_success:
- return None
+ self._raise_http(resp)
body = resp.json()
teams = body if isinstance(body, list) else body.get("teams") or body.get("data") or []
- for t in teams:
- if isinstance(t, dict) and t.get("team_alias") == project:
- return t.get("team_id")
- if hasattr(t, "team_alias") and getattr(t, "team_alias", None) == project:
- return getattr(t, "team_id", None)
- return None
+ if not isinstance(teams, list):
+ teams = []
+ return self._ingest_team_list(teams)
+
+ @staticmethod
+ def _team_info_from_row(row: Dict[str, Any]) -> TeamInfo:
+ return TeamInfo(
+ str(row.get("team_id") or ""),
+ float(row.get("max_budget") or 0),
+ float(row.get("spend") or 0),
+ )
- async def _ensure_team_id(self, project: str) -> str:
- """Map LDAP project (team_alias) → LiteLLM team_id; create team if needed."""
- budget_usd = float(self._cfg.budgets.default_team_budget_usd)
- duration = self._cfg.budgets.duration
+ async def warm(self) -> None:
+ """Load project→team_id from LiteLLM. Fail-fast if the proxy is unreachable."""
+ await self._team_list_rows()
- existing = await self.team_info(project)
- if existing:
- return existing.team_id
+ async def ensure_team_id(self, project: str) -> str:
+ """Map LDAP project (team_alias) → LiteLLM team_id; create team if needed."""
+ project = (project or "").strip()
+ if not project:
+ raise BackendUnavailable("project is required")
+ if project in self._team_ids:
+ return self._team_ids[project]
- found = await self._find_team_id_by_alias(project)
- if found:
- self._remember_team(project, found, budget_usd)
- return found
+ rows = await self._team_list_rows()
+ if project in self._team_ids:
+ return self._team_ids[project]
+ budget_usd = float(self._cfg.budgets.default_team_budget_usd)
+ duration = self._cfg.budgets.duration
resp = await self._request(
"POST",
"team/new",
@@ -242,38 +260,50 @@ class LiteLLMBackend:
},
)
if resp.status_code >= 400:
- found = await self._find_team_id_by_alias(project)
- if found:
- self._remember_team(project, found, budget_usd)
- return found
+ await self._team_list_rows()
+ if project in self._team_ids:
+ return self._team_ids[project]
self._raise_http(resp)
team_id = resp.json().get("team_id")
if not team_id:
raise BackendUnavailable("LiteLLM team/new returned no team_id")
- self._remember_team(project, team_id, budget_usd)
+ self._team_ids[project] = str(team_id)
return str(team_id)
async def team_info(self, project: str) -> Optional[TeamInfo]:
- t = self._store.snapshot().get("teams", {}).get(project)
- if not t:
+ """Live team spend/budget. Cache holds ids only; spend is never cached.
+
+ Cache hit → team/info. Cache miss → team/list (fills cache + returns
+ list-row fields; no second team/info).
+ """
+ project = (project or "").strip()
+ if not project:
return None
- spend = t.get("spend", 0.0)
- max_budget = t.get("max_budget", 0.0)
- try:
+
+ if project in self._team_ids:
+ team_id = self._team_ids[project]
resp = await self._request(
"GET",
"team/info",
- params={"team_id": t["team_id"]},
+ params={"team_id": team_id},
+ )
+ self._raise_http(resp)
+ body = resp.json()
+ info = body.get("team_info") or body
+ if not isinstance(info, dict):
+ info = {}
+ return TeamInfo(
+ team_id,
+ float(info.get("max_budget") or 0),
+ float(info.get("spend") or 0),
)
- if resp.is_success:
- info = resp.json().get("team_info") or resp.json()
- if isinstance(info, dict):
- spend = info.get("spend", spend)
- max_budget = info.get("max_budget", max_budget)
- except BackendUnavailable:
- pass
- return TeamInfo(t["team_id"], float(max_budget or 0), float(spend or 0))
+
+ rows = await self._team_list_rows()
+ for row in rows:
+ if str(row.get("team_alias") or "") == project:
+ return self._team_info_from_row(row)
+ return None
async def list_keys(
self,
@@ -290,11 +320,9 @@ class LiteLLMBackend:
if user:
params["user_id"] = user
if project:
- # Resolve project → team_id for LiteLLM filter (no ensure/create).
- team_id = await self._find_team_id_by_alias(project)
- if not team_id:
- cached = self._store.snapshot().get("teams", {}).get(project)
- team_id = cached.get("team_id") if cached else None
+ if project not in self._team_ids:
+ await self._team_list_rows()
+ team_id = self._team_ids.get(project)
if not team_id:
return []
params["team_id"] = team_id
@@ -320,7 +348,7 @@ class LiteLLMBackend:
project = (project or "").strip()
if not project:
raise BackendUnavailable("create_key requires project")
- team_id = await self._ensure_team_id(project)
+ team_id = await self.ensure_team_id(project)
meta = dict(metadata or {})
meta["project"] = project
payload: Dict[str, Any] = {
@@ -366,11 +394,5 @@ class LiteLLMBackend:
self._raise_http(resp)
async def usage(self, project: Optional[str]) -> List[Dict]:
- rows = self._store.snapshot().get("usage", [])
- if project is None:
- return list(rows)
- return [r for r in rows if r.get("project") == project]
-
-
-def make_backend(cfg: Any, store: StateStore) -> Backend:
- return LiteLLMBackend(cfg, store)
+ # Spend APIs not wired yet.
+ return []
diff --git a/llmao/store.py b/llmao/store.py
deleted file mode 100644
index c47f836..0000000
--- a/llmao/store.py
+++ /dev/null
@@ -1,44 +0,0 @@
-"""Minimal JSON-file state store.
-
-LiteLLMBackend uses this as a local project→team_id cache (and legacy usage
-rows). Unit tests use MockBackend’s in-memory dict, not this store.
-"""
-from __future__ import annotations
-
-import json
-import os
-import threading
-from typing import Any, Callable, Dict
-
-
-class StateStore:
- def __init__(self, path: str):
- self._path = path
- self._lock = threading.RLock()
- if not os.path.exists(path):
- self._write({"teams": {}, "usage": []})
-
- def _read(self) -> Dict[str, Any]:
- try:
- with open(self._path, "r", encoding="utf-8") as fh:
- return json.load(fh)
- except (FileNotFoundError, json.JSONDecodeError):
- return {"teams": {}, "usage": []}
-
- def _write(self, data: Dict[str, Any]) -> None:
- tmp = f"{self._path}.tmp"
- with open(tmp, "w", encoding="utf-8") as fh:
- json.dump(data, fh, indent=2, default=str)
- os.replace(tmp, self._path)
-
- def snapshot(self) -> Dict[str, Any]:
- with self._lock:
- return self._read()
-
- def update(self, fn: Callable[[Dict[str, Any]], Any]) -> Any:
- """Run ``fn`` against the loaded state under lock, persist, return fn's result."""
- with self._lock:
- data = self._read()
- result = fn(data)
- self._write(data)
- return result
diff --git a/main.py b/main.py
index 0afcb8c..054e6c7 100644
--- a/main.py
+++ b/main.py
@@ -50,9 +50,8 @@ def create_app():
)
from llmao.auth import make_token_handler
- from llmao.litellm_client import make_backend
+ from llmao.litellm_client import LiteLLMBackend
from llmao.seam import Seam
- from llmao.store import StateStore
# Config is app.cfg (EasyDict from config.yaml) — dotted access throughout.
from llmao.models import load_model_list
@@ -60,8 +59,7 @@ def create_app():
# Fail-fast: model_list.yaml is required (same presumption as config.yaml).
load_model_list(cfg=app.cfg)
- store = StateStore(app.cfg.state_path)
- backend = make_backend(app.cfg, store)
+ backend = LiteLLMBackend(app.cfg)
seam = Seam(app.cfg, backend)
app.token_handler = make_token_handler(app.cfg)
app.config["LLMAO_SEAM"] = seam
@@ -83,6 +81,15 @@ def create_app():
return resp
return exc
+ @app.before_serving
+ async def _warm_backend():
+ # Fail-fast if LiteLLM is down (puppet starts proxy first; local: make proxy).
+ await backend.warm()
+ _LOGGER.info(
+ "LiteLLM team cache warmed (%d project→team_id mappings)",
+ len(backend._team_ids),
+ )
+
@app.after_serving
async def _close_backend():
await backend.aclose()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org