You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/12 22:01:56 UTC
(tooling-llmao) branch main updated: requests: keep access requests and tier overrides in LiteLLM's memory store
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
The following commit(s) were added to refs/heads/main by this push:
new 988b2a0 requests: keep access requests and tier overrides in LiteLLM's memory store
988b2a0 is described below
commit 988b2a040d387bf8db826cba5c46be1dff73c057
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Mon Oct 12 17:01:01 2026 -0500
requests: keep access requests and tier overrides in LiteLLM's memory store
Both lived in team metadata. Listing requests meant reading every team,
which is why My Keys called team/list at all, and answering one rewrote a
team's whole metadata blob.
They now go in /v1/memory: one row per request, keyed llmao:request:<id>,
and one row for the tier limits an admin has changed, llmao:tiers. Listing
is a prefix query, and a decision writes one row. Nothing is cached -- there
are a handful of rows and a handful of people -- and nothing is migrated,
because nothing was stored yet.
Request becomes AccessRequest, in llmao/access_requests.py. In a web
application talking to an HTTP API, a bare Request means three other things
first.
warm() now also reads the week's usage, so the six days that are already
history are loaded once at startup rather than by the first visitor.
Per-project token caps are still team metadata.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
docs/ARCHITECTURE.md | 5 +
llmao/{requests.py => access_requests.py} | 99 +++++------
llmao/litellm_client.py | 174 +++++++++---------
llmao/seam.py | 20 +--
pages.py | 3 +-
.../{test_requests.py => test_access_requests.py} | 198 ++++++++++++++++-----
6 files changed, 315 insertions(+), 184 deletions(-)
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 5d16f2d..08ceb05 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -52,6 +52,11 @@ production Puppet restarts the service after a change.
Developers: system PostgreSQL + `make db`. Production: Puppet + on-disk
`database_url`.
+**LLMAO's own records** live in LiteLLM's memory store (`/v1/memory`), since
+LLMAO has no database: one row per access request (`llmao:request:<id>`) and
+one for the tier limits an admin has changed (`llmao:tiers`). Per-project
+token caps are still team metadata.
+
**PATs:** personal keys bind ASF uid + project team + purpose; automation
keys are team-scoped exceptions (who may create them is an **open RAI
policy** question — see design + `docs/STATUS.md`). Secrets shown once;
diff --git a/llmao/requests.py b/llmao/access_requests.py
similarity index 77%
rename from llmao/requests.py
rename to llmao/access_requests.py
index 91607a7..a4f55a0 100644
--- a/llmao/requests.py
+++ b/llmao/access_requests.py
@@ -31,37 +31,33 @@ answer is visible where the asking happened.
WHERE THIS LIVES
-In LiteLLM team metadata, beside grantor and token_cap. llmao has no database
-of its own and docs/ARCHITECTURE.md states that as a property, so a request
-is a row on a team rather than a row in a table llmao owns.
-
-That is the ugly part and it is deliberate: listing everything pending means
-reading every team. warm() already enumerates them once per boot, and at
-twenty projects the cost is noise. At three hundred it would not be, and the
-shape here -- a dataclass with to_dict/from_dict and a store that only knows
-how to list, add and replace -- is meant to be liftable into a table without
-touching anything that calls it.
-
-A request with no project lives on the reserved entitlement-defaults team,
-which already exists for tier overrides.
-
-CONCURRENCY
-
-Writes are read-modify-write on the whole metadata blob. Two admins answering
-different requests in the same second can lose one. At pilot volume that is
-theoretical; if it stops being theoretical, that is the signal to lift this
-into a table rather than to add a lock here.
+In LiteLLM's memory store (/v1/memory), one row per request, keyed
+`llmao:request:<id>` with the request as JSON. llmao has no database of its
+own and docs/ARCHITECTURE.md states that as a property; the memory store is
+LiteLLM's, and a request is a row in it.
+
+They first lived in team metadata, beside grantor and token_cap. That made
+listing them a read of every team and every decision a rewrite of a team's
+whole metadata blob. A row each has neither cost: listing is one prefix
+query, and answering one request cannot lose another.
+
+NAMING
+
+AccessRequest, not AccessRequest. This process is a web application talking to an
+HTTP API, and a bare `AccessRequest` there means three other things first.
"""
from __future__ import annotations
import datetime as dt
+import json
import secrets
from dataclasses import dataclass, field
from typing import Any
-# Metadata key on the team. One list, newest last.
-METADATA_KEY = "requests"
+# Every request's key in the memory store starts with this, and listing them
+# is a prefix query on it.
+KEY_PREFIX = "llmao:request:"
KIND_CAPACITY = "capacity"
KIND_KEY = "key"
@@ -89,7 +85,7 @@ def _parse(stamp: str | None) -> dt.datetime | None:
@dataclass
-class Request:
+class AccessRequest:
"""One ask, and its answer.
`granted` carries what an admin actually approved, which is not always
@@ -118,7 +114,9 @@ class Request:
consumed_at: str = ""
@classmethod
- def new(cls, *, kind: str, requester: str, project: str | None, reason: str, wanted: dict[str, Any]) -> Request:
+ def new(
+ cls, *, kind: str, requester: str, project: str | None, reason: str, wanted: dict[str, Any]
+ ) -> AccessRequest:
if kind not in (KIND_CAPACITY, KIND_KEY):
raise ValueError(f"unknown request kind: {kind}")
requester = (requester or "").strip()
@@ -219,11 +217,11 @@ class Request:
}
@classmethod
- def from_dict(cls, d: dict[str, Any]) -> Request | None:
+ def from_dict(cls, d: dict[str, Any]) -> AccessRequest | None:
"""Rehydrate, or None if the row is not a request.
- Returns None rather than raising: metadata is shared with other
- things, and one malformed entry must not cost the whole queue.
+ Returns None rather than raising: one malformed entry must not cost
+ the whole queue.
"""
if not isinstance(d, dict) or not d.get("id") or not d.get("kind"):
return None
@@ -244,38 +242,37 @@ class Request:
)
-def load(metadata: dict[str, Any] | None) -> list[Request]:
- """Requests stored on one team's metadata."""
- raw = (metadata or {}).get(METADATA_KEY)
- if not isinstance(raw, list):
- return []
- out = []
- for item in raw:
- req = Request.from_dict(item)
- if req is not None:
- out.append(req)
- return out
+def key_for(request_id: str) -> str:
+ """The memory-store key for one request."""
+ return KEY_PREFIX + request_id
+
+def dumps(req: AccessRequest) -> str:
+ """A request as the memory store holds it."""
+ return json.dumps(req.to_dict(), sort_keys=True)
-def dump(requests: list[Request]) -> list[dict[str, Any]]:
- return [r.to_dict() for r in requests]
+def loads(value: Any) -> AccessRequest | None:
+ """Rehydrate a stored value, or None if it is not a request.
-def upsert(requests: list[Request], req: Request) -> list[Request]:
- """Replace by id, or append. Returns a new list."""
- out = [r for r in requests if r.id != req.id]
- out.append(req)
- return out
+ None rather than raising: the store is shared with other things, and one
+ malformed row must not cost the whole queue.
+ """
+ try:
+ return AccessRequest.from_dict(json.loads(value))
+ except (TypeError, ValueError):
+ return None
-def prune(requests: list[Request], *, keep_decided: int = 50, now: dt.datetime | None = None) -> list[Request]:
+def prune(
+ requests: list[AccessRequest], *, keep_decided: int = 50, now: dt.datetime | None = None
+) -> list[AccessRequest]:
"""Drop expired approvals and trim decided history.
- Metadata is a JSON blob on a row, not a table -- it grows forever if
- nothing trims it, and every read of the team carries it. Pending requests
- are always kept; decided ones are capped at the most recent `keep_decided`
- so the audit trail survives a pilot without the blob becoming the largest
- thing in the database.
+ Nothing else trims the store, and every listing carries all of it.
+ Pending requests are always kept; decided ones are capped at the most
+ recent `keep_decided`, which is plenty of audit trail for a pilot. The
+ caller deletes whatever this leaves out.
"""
pending, decided = [], []
for r in requests:
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index 8e16c1b..065663b 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -28,6 +28,7 @@ from __future__ import annotations
import asyncio
import datetime as _dt
+import json
import logging
import pathlib
import re
@@ -39,7 +40,7 @@ from urllib.parse import urlparse
import httpx
from asfpy.stopwatch import Stopwatch
-from llmao import requests as _requests
+from llmao import access_requests as _access
from llmao.fleet import LEGACY_SERVERS, ExtraLiteLLM, normalize_peer_ip, snapshot_for_status
from llmao.model_status import (
HEALTHY,
@@ -413,14 +414,10 @@ class LiteLLMBackend:
self._team_ids[str(alias)] = str(tid)
return rows
- # Requests live in team metadata, so listing them reads every team. My
- # Keys does that twice per load -- once for pending approvals, once for
- # the person's own asks -- on top of two spend-log scans and a team_info
- # per project.
- #
- # Sixty seconds: long enough that a page load is one scan rather than
- # several, short enough that an approval appears while the admin who
- # granted it is still looking at the page.
+ # The listing is every team with its budget and metadata, and it is the
+ # slowest admin call left. Sixty seconds: long enough that a page load
+ # is one listing rather than several, short enough that a cap an admin
+ # has just changed shows while they are still looking at the page.
_TEAM_TTL_S = 60
async def _team_list_rows(self, *, fresh: bool = False) -> list[dict[str, Any]]:
@@ -465,6 +462,13 @@ class LiteLLMBackend:
rows = await self._team_list_rows(fresh=True)
await self._backfill_token_caps(rows)
await self.ensure_commercial()
+ # Six of the week's seven days are history and are never asked for
+ # again, so reading them here means no visitor waits for them. Not
+ # worth refusing to start over: a cold cache is only slow.
+ try:
+ await self._daily_rows(self._SPEND_DAYS)
+ except BackendUnavailableError as e:
+ _LOGGER.warning("could not warm usage: %s", e)
async def _backfill_token_caps(self, rows: list[dict[str, Any]]) -> None:
"""Give every team without a token allocation the configured default.
@@ -587,82 +591,100 @@ class LiteLLMBackend:
team_id=str(team_id),
)
- # Runtime tier overrides live on a reserved LiteLLM team rather than in
- # llmao's own state: llmao has no database, and the only thing worse than
- # a missing override is one that disagrees between two instances.
+ # The team a key with no project is minted into. LiteLLM wants every
+ # key in a team, and a projectless one has to go somewhere.
_TIER_STORE_TEAM = "llmao-entitlement-defaults"
- # ---- requests ------------------------------------------------------
+ # ---- LiteLLM's memory store -----------------------------------------
#
- # Stored on team metadata beside grantor and token_cap. llmao has no
- # database of its own and ARCHITECTURE.md states that as a property, so a
- # request is a row on a team rather than a row in a table llmao owns.
+ # llmao has no database of its own, and ARCHITECTURE.md states that as a
+ # property. What it needs to remember -- requests awaiting an answer, and
+ # the tier limits an admin has changed -- goes in LiteLLM's /v1/memory: a
+ # key, a string value, one row each.
#
- # The cost is that listing everything pending reads every team. warm()
- # already enumerates them once per boot and at twenty projects this is
- # noise; at three hundred it would not be, and that is the signal to lift
- # it into a table. Nothing above this layer would have to change.
+ # Both used to live in team metadata, which made listing requests a read
+ # of every team and every write a rewrite of a team's whole blob.
+ #
+ # Nothing here is cached. There are a handful of rows and a handful of
+ # people; asking each time is quick, and leaves nothing to go stale.
+ _TIERS_KEY = "llmao:tiers"
- async def _team_metadata(self, project: str) -> tuple[str, dict[str, Any]]:
- info = await self.ensure_team(project)
- return info.team_id, dict(info.metadata or {})
+ async def _memory_get(self, key: str) -> str | None:
+ resp = await self._request("GET", f"v1/memory/{key}")
+ if resp.status_code == 404:
+ return None
+ self._raise_http(resp)
+ body = resp.json()
+ return body.get("value") if isinstance(body, dict) else None
- async def add_request(self, req: Any) -> Any:
- """Store a new request, or persist a decision on an existing one.
+ async def _memory_put(self, key: str, value: str) -> None:
+ resp = await self._request("PUT", f"v1/memory/{key}", json={"value": value})
+ self._raise_http(resp)
- Projectless requests live on the reserved entitlement-defaults team,
- which already exists for tier overrides.
+ async def _memory_delete(self, key: str) -> None:
+ resp = await self._request("DELETE", f"v1/memory/{key}")
+ if resp.status_code != 404:
+ self._raise_http(resp)
+
+ async def _memory_list(self, prefix: str) -> list[dict[str, Any]]:
+ """Every row whose key starts with `prefix`, in one page.
- Read-modify-write on the whole metadata blob: two admins answering
- different requests in the same second can lose one. At pilot volume
- that is theoretical, and if it stops being so that is a reason to
- move this into a table rather than to add a lock here.
+ Outstanding requests are counted on one hand, and add_request trims
+ the decided ones, so a page is far more than there will ever be.
"""
- project = req.project or self._TIER_STORE_TEAM
- team_id, meta = await self._team_metadata(project)
- merged = _requests.upsert(_requests.load(meta), req)
- meta[_requests.METADATA_KEY] = _requests.dump(_requests.prune(merged))
- resp = await self._request(
- "POST",
- "team/update",
- json={"team_id": team_id, "metadata": meta},
- )
+ resp = await self._request("GET", "v1/memory", params={"key_prefix": prefix, "page_size": 500})
self._raise_http(resp)
- # The listing carries metadata, so a request just written would not
- # appear until the TTL expired -- which reads as the form having
- # silently failed.
- self._team_rows_cache = None
+ body = resp.json()
+ rows = body.get("memories") if isinstance(body, dict) else None
+ return [r for r in rows or [] if isinstance(r, dict)]
+
+ # ---- requests ------------------------------------------------------
+
+ async def add_request(self, req: Any) -> Any:
+ """Store a new request, or persist a decision on an existing one.
+
+ One row per request, so answering one cannot lose another.
+ """
+ await self._memory_put(_access.key_for(req.id), _access.dumps(req))
+
+ # Nothing else trims the store, so do it while writing: a lapsed
+ # approval, or a decision older than the history we keep, goes. The
+ # request is already saved; failing to tidy is no reason to say it
+ # was not.
+ try:
+ stored = await self.list_requests()
+ kept = _access.prune(stored)
+ for old in stored:
+ if old not in kept:
+ await self._memory_delete(_access.key_for(old.id))
+ except BackendUnavailableError as e:
+ _LOGGER.warning("could not trim stored requests: %s", e)
return req
# Same operation; named for what the caller means.
save_request = add_request
async def list_requests(self, project: str | None = None) -> list[Any]:
- """Requests for one project, or every one across the fleet.
-
- With no project this reads every team. That is the ugly part of
- storing them in metadata, and it is bounded by the number of PMCs
- rather than by the number of requests.
- """
- if project is not None:
- _, meta = await self._team_metadata(project)
- return _requests.load(meta)
-
- out: list[Any] = []
- for row in await self._team_list_rows():
- if not isinstance(row, dict):
- continue
- meta = row.get("metadata")
- if not isinstance(meta, dict) or _requests.METADATA_KEY not in meta:
- continue
- out.extend(_requests.load(meta))
+ """Requests for one project, or every one."""
+ out = []
+ for row in await self._memory_list(_access.KEY_PREFIX):
+ req = _access.loads(row.get("value"))
+ if req is not None and (project is None or req.project == project):
+ out.append(req)
return out
async def find_request(self, request_id: str) -> Any | None:
- for r in await self.list_requests():
- if r.id == request_id:
- return r
- return None
+ return _access.loads(await self._memory_get(_access.key_for(request_id)))
+
+ # ---- tier overrides ------------------------------------------------
+
+ async def _tier_overrides(self) -> dict[str, Any]:
+ """{tier: {limit: value}} as an admin has set them. Empty until one has."""
+ try:
+ held = json.loads(await self._memory_get(self._TIERS_KEY) or "{}")
+ except ValueError:
+ return {}
+ return held if isinstance(held, dict) else {}
async def tier_entitlement(self, tier: str) -> dict[str, Any]:
"""A tier's effective limits: config, with any admin override on top.
@@ -679,12 +701,9 @@ class LiteLLMBackend:
effective[name] = val
try:
- info = await self.team_info(self._TIER_STORE_TEAM)
+ stored = (await self._tier_overrides()).get(tier)
except BackendUnavailableError:
return effective
- if info is None:
- return effective
- stored = (info.metadata or {}).get(tier)
if isinstance(stored, dict):
effective.update({k: v for k, v in stored.items() if v is not None})
return effective
@@ -733,17 +752,10 @@ class LiteLLMBackend:
if int(val) < 0:
raise BackendUnavailableError(f"{name} must not be negative")
- team = await self.ensure_team(self._TIER_STORE_TEAM)
- meta = dict(team.metadata or {})
- meta[tier] = {**(meta.get(tier) or {}), **changes}
-
- resp = await self._request(
- "POST",
- "team/update",
- json={"team_id": team.team_id, "metadata": meta},
- )
- self._raise_http(resp)
- self._team_rows_cache = None
+ # Every tier shares the one row, so this is read, change, write.
+ tiers = await self._tier_overrides()
+ tiers[tier] = {**(tiers.get(tier) or {}), **changes}
+ await self._memory_put(self._TIERS_KEY, json.dumps(tiers, sort_keys=True))
_LOGGER.info("tier %s updated: %s", tier, sorted(changes))
# The cap is re-read on every check, but today's usage is cached --
# without this an admin lowering a cap sees a stale remaining figure
diff --git a/llmao/seam.py b/llmao/seam.py
index 8f5e67d..859a961 100644
--- a/llmao/seam.py
+++ b/llmao/seam.py
@@ -30,7 +30,7 @@ import logging
from dataclasses import dataclass, field
from typing import Any
-from . import requests as _requests
+from . import access_requests as _access
from .litellm_client import (
GRANTOR_FREE_TIER,
TIER_FREE,
@@ -288,8 +288,8 @@ class Seam:
# The approver's first question is why. Asking for it here saves
# a round trip and makes the queue readable.
raise AuthzError("Say what the extra capacity is for.")
- req = _requests.Request.new(
- kind=_requests.KIND_CAPACITY,
+ req = _access.AccessRequest.new(
+ kind=_access.KIND_CAPACITY,
requester=identity.uid,
project=project,
reason=reason,
@@ -319,8 +319,8 @@ class Seam:
raise AuthzError("Say what the key is for.")
if project and project not in (identity.committees or []):
raise AuthzError(f"You are not on the {project} PMC.")
- req = _requests.Request.new(
- kind=_requests.KIND_KEY,
+ req = _access.AccessRequest.new(
+ kind=_access.KIND_KEY,
requester=identity.uid,
project=project,
reason=reason,
@@ -372,7 +372,7 @@ class Seam:
"""
if not identity.is_site_admin:
raise AuthzError("The request queue is limited to site admins.")
- pending = [r for r in await self._backend.list_requests() if r.state == _requests.STATE_PENDING]
+ pending = [r for r in await self._backend.list_requests() if r.state == _access.STATE_PENDING]
return sorted(pending, key=lambda r: r.created_at or "")
async def decide_request(
@@ -394,12 +394,12 @@ class Seam:
if req is None:
raise AuthzError("No such request.")
req.decide(
- state=_requests.STATE_APPROVED if approve else _requests.STATE_DENIED,
+ state=_access.STATE_APPROVED if approve else _access.STATE_DENIED,
by=identity.uid,
reason=reason,
granted=granted,
)
- if approve and req.kind == _requests.KIND_CAPACITY:
+ if approve and req.kind == _access.KIND_CAPACITY:
await self._backend.update_team_entitlement(req.project, token_cap=int(req.granted.get("token_cap") or 0))
return await self._backend.save_request(req)
@@ -473,8 +473,8 @@ class Seam:
"""Approvals from an already-fetched list.
Same filter as my_key_approvals, without the read. My Keys needs both
- this and the person's own requests, and listing requests reads every
- team -- so the page fetches once and derives both.
+ this and the person's own requests, so the page fetches once and
+ derives both.
"""
return [
{
diff --git a/pages.py b/pages.py
index 09309e0..50ae30e 100644
--- a/pages.py
+++ b/pages.py
@@ -1195,8 +1195,7 @@ async def keys_list(result):
except BackendUnavailableError:
result.projects = []
- # One read, two views. my_key_approvals and my_requests both list every
- # request; asking twice doubled the most expensive thing on the page.
+ # One read, two views: the approvals are a filter of the same list.
try:
mine = await APP.seam.my_requests(ident)
except BackendUnavailableError:
diff --git a/tests/test_requests.py b/tests/test_access_requests.py
similarity index 57%
rename from tests/test_requests.py
rename to tests/test_access_requests.py
index 2cac6cd..8061d42 100644
--- a/tests/test_requests.py
+++ b/tests/test_access_requests.py
@@ -21,24 +21,27 @@ The failures worth guarding are the quiet ones -- an approval that can be
used twice, one that never lapses, or a denial with no explanation attached.
"""
+import asyncio
import datetime as dt
+import json
+import types
import pytest
-from llmao.requests import (
+from llmao import access_requests
+from llmao.access_requests import (
APPROVAL_TTL_DAYS,
KIND_CAPACITY,
KIND_KEY,
- METADATA_KEY,
STATE_APPROVED,
STATE_CONSUMED,
STATE_DENIED,
STATE_PENDING,
- Request,
- dump,
- load,
+ AccessRequest,
+ dumps,
+ key_for,
+ loads,
prune,
- upsert,
)
UTC = dt.UTC
@@ -53,7 +56,7 @@ def _capacity(**kw):
"wanted": {"token_cap": 40_000_000},
}
base.update(kw)
- return Request.new(**base)
+ return AccessRequest.new(**base)
def _key(**kw):
@@ -65,7 +68,7 @@ def _key(**kw):
"wanted": {"purpose": "scrutineer-ci", "tier": "service"},
}
base.update(kw)
- return Request.new(**base)
+ return AccessRequest.new(**base)
# ---------------------------------------------------------------------------
@@ -98,7 +101,7 @@ def test_a_key_request_does_not_need_one():
def test_an_unknown_kind_is_refused():
with pytest.raises(ValueError):
- Request.new(kind="whatever", requester="akm", project="p", reason="", wanted={})
+ AccessRequest.new(kind="whatever", requester="akm", project="p", reason="", wanted={})
# ---------------------------------------------------------------------------
@@ -204,45 +207,27 @@ def test_a_capacity_approval_does_not_expire():
# ---------------------------------------------------------------------------
-# Storage on team metadata
+# Storage in LiteLLM's memory store
# ---------------------------------------------------------------------------
-def test_round_trips_through_metadata():
+def test_round_trips_through_a_stored_value():
r = _capacity()
r.decide(state=STATE_APPROVED, by="admin")
- meta = {"grantor": "Free Tier", METADATA_KEY: dump([r])}
- back = load(meta)
- assert len(back) == 1
- assert back[0].to_dict() == r.to_dict()
-
-
-def test_load_ignores_anything_that_is_not_a_request():
- """Metadata is shared. One malformed entry must not cost the queue."""
- meta = {
- METADATA_KEY: [
- {"id": "x", "kind": KIND_KEY, "requester": "a"},
- {"no": "id"},
- "a string",
- None,
- ]
- }
- assert len(load(meta)) == 1
+ back = loads(dumps(r))
+ assert back.to_dict() == r.to_dict()
-def test_load_on_metadata_without_requests():
- assert load({"grantor": "Free Tier"}) == []
- assert load(None) == []
+def test_loads_ignores_anything_that_is_not_a_request():
+ """The store is shared. One malformed row must not cost the queue."""
+ assert loads(json.dumps({"id": "x", "kind": KIND_KEY, "requester": "a"})) is not None
+ for junk in (json.dumps({"no": "id"}), json.dumps("a string"), "not json at all", None):
+ assert loads(junk) is None
-def test_upsert_replaces_by_id():
- r = _capacity()
- stored = [r]
- r2 = Request.from_dict(r.to_dict())
- r2.decide(state=STATE_APPROVED, by="admin")
- out = upsert(stored, r2)
- assert len(out) == 1
- assert out[0].state == STATE_APPROVED
+def test_every_request_key_shares_the_prefix_listing_asks_for():
+ assert key_for("abc") == "llmao:request:abc"
+ assert key_for("abc").startswith(access_requests.KEY_PREFIX)
def test_prune_keeps_pending_and_drops_expired_approvals():
@@ -255,7 +240,7 @@ def test_prune_keeps_pending_and_drops_expired_approvals():
def test_prune_caps_decided_history():
- """Metadata is a blob on a row and every read of the team carries it."""
+ """Nothing else trims the store, and every listing carries all of it."""
reqs = []
for i in range(80):
r = _capacity()
@@ -319,3 +304,136 @@ def test_an_adjusted_request_differs_from_what_was_asked():
assert r.granted != r.wanted
assert r.granted["token_cap"] == 30_000_000
assert r.wanted["token_cap"] == 40_000_000
+
+
+# ---------------------------------------------------------------------------
+# The backend, against a memory store that behaves like LiteLLM's
+#
+# Requests and tier overrides used to live in team metadata. The point of
+# moving them is that reading them no longer touches a team at all, so these
+# fail on any call that is not to /v1/memory.
+# ---------------------------------------------------------------------------
+
+
+def _backend(*, down=False):
+ from llmao.litellm_client import BackendUnavailableError, LiteLLMBackend
+
+ b = object.__new__(LiteLLMBackend)
+ b._cfg = types.SimpleNamespace(
+ entitlements=types.SimpleNamespace(
+ default=types.SimpleNamespace(rpm_limit=60, tpm_limit=60000, max_parallel_requests=4),
+ free_tier=types.SimpleNamespace(token_cap=1_000_000, token_window_days=7),
+ )
+ )
+ b._daily_at = None
+ b.store = {}
+ b.calls = []
+
+ async def fake_request(method, path, params=None, json=None, **kw):
+ b.calls.append((method, path))
+ if down:
+ raise BackendUnavailableError("could not reach LiteLLM")
+ assert path == "v1/memory" or path.startswith("v1/memory/"), f"unexpected call: {method} {path}"
+ key = path.removeprefix("v1/memory").lstrip("/")
+
+ def reply(status, body):
+ return types.SimpleNamespace(status_code=status, content=b"x", json=lambda: body, text=str(body))
+
+ if method == "GET" and not key:
+ hits = [{"key": k, "value": v} for k, v in b.store.items() if k.startswith(params["key_prefix"])]
+ return reply(200, {"memories": hits, "total": len(hits)})
+ if method == "GET":
+ return reply(200, {"key": key, "value": b.store[key]}) if key in b.store else reply(404, {})
+ if method == "PUT":
+ b.store[key] = json["value"]
+ return reply(200, {"key": key, "value": json["value"]})
+ if method == "DELETE":
+ return reply(200, {"deleted": True}) if b.store.pop(key, None) is not None else reply(404, {})
+ raise AssertionError(f"unexpected call: {method} {path}")
+
+ def raise_http(resp):
+ if resp.status_code >= 400:
+ raise BackendUnavailableError(f"LiteLLM admin error {resp.status_code}")
+
+ b._request = fake_request
+ b._raise_http = raise_http
+ return b
+
+
+def _run(coro):
+ return asyncio.run(coro)
+
+
+def test_a_stored_request_is_listed_and_found():
+ b = _backend()
+ r = _key()
+ _run(b.add_request(r))
+ assert list(b.store) == [f"llmao:request:{r.id}"]
+ assert [x.id for x in _run(b.list_requests())] == [r.id]
+ assert _run(b.find_request(r.id)).to_dict() == r.to_dict()
+ assert _run(b.find_request("no-such-id")) is None
+
+
+def test_listing_for_a_project_leaves_the_others_out():
+ b = _backend()
+ mine, theirs, nobodys = _capacity(project="tooling"), _capacity(project="mahout"), _key()
+ for r in (mine, theirs, nobodys):
+ _run(b.add_request(r))
+ assert [x.id for x in _run(b.list_requests("tooling"))] == [mine.id]
+ assert len(_run(b.list_requests())) == 3
+
+
+def test_a_decision_rewrites_one_row_and_no_other():
+ """In team metadata, answering one request rewrote all of them."""
+ b = _backend()
+ first, second = _capacity(), _capacity()
+ _run(b.add_request(first))
+ _run(b.add_request(second))
+ untouched = b.store[key_for(second.id)]
+
+ found = _run(b.find_request(first.id))
+ found.decide(state=STATE_APPROVED, by="admin")
+ _run(b.save_request(found))
+
+ assert _run(b.find_request(first.id)).state == STATE_APPROVED
+ assert b.store[key_for(second.id)] == untouched
+ assert len(b.store) == 2
+
+
+def test_a_lapsed_approval_is_deleted_on_the_next_write():
+ b = _backend()
+ stale = _key()
+ stale.decide(state=STATE_APPROVED, by="admin")
+ long_ago = dt.datetime.now(UTC) - dt.timedelta(days=APPROVAL_TTL_DAYS + 1)
+ stale.decided_at = long_ago.isoformat(timespec="seconds")
+ b.store[key_for(stale.id)] = dumps(stale)
+
+ fresh = _capacity()
+ _run(b.add_request(fresh))
+ assert list(b.store) == [key_for(fresh.id)]
+
+
+def test_tier_limits_are_config_until_an_admin_changes_them():
+ b = _backend()
+ assert _run(b.tier_entitlement("free_tier"))["token_cap"] == 1_000_000
+ got = _run(b.set_tier_entitlement("free_tier", token_cap=250_000))
+ assert got["token_cap"] == 250_000
+ assert got["token_window_days"] == 7, "what was not changed still comes from config"
+ assert list(b.store) == ["llmao:tiers"]
+
+
+def test_changing_one_tier_leaves_another_alone():
+ """Every tier shares the one row."""
+ b = _backend()
+ _run(b.set_tier_entitlement("free_tier", token_cap=250_000))
+ _run(b.set_tier_entitlement("service", rpm_limit=10))
+ _run(b.set_tier_entitlement("free_tier", token_window_days=3))
+ assert json.loads(b.store["llmao:tiers"]) == {
+ "free_tier": {"token_cap": 250_000, "token_window_days": 3},
+ "service": {"rpm_limit": 10},
+ }
+
+
+def test_tier_limits_fall_back_to_config_when_litellm_is_down():
+ """A page must still have limits to show during a proxy restart."""
+ assert _run(_backend(down=True).tier_entitlement("free_tier"))["token_cap"] == 1_000_000
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org