(tooling-llmao) 08/17: Self-host only catalog; add litellm Postgres; bump vLLM to 0.11.0

[ak...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git

commit 6576472de8c5b89f1321e863460e0f6469069366
Author: Andrew Musselman <an...@gmail.com>
AuthorDate: Tue Jul 7 16:44:26 2026 -0700

    Self-host only catalog; add litellm Postgres; bump vLLM to 0.11.0
---
 infra/docker/docker-compose.yml | 34 ++++++++++++++++++++++++++++++++--
 llmao/catalog.py                |  1 +
 llmao/litellm_client.py         |  8 +++-----
 3 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml
index 55b540e..699d08c 100644
--- a/infra/docker/docker-compose.yml
+++ b/infra/docker/docker-compose.yml
@@ -18,7 +18,7 @@
 # capped per model so they co-reside. Tune the caps against real VRAM use.
 
 x-vllm-common: &vllm-common
-  image: vllm/vllm-openai:v0.6.6   # pin; never :latest
+  image: vllm/vllm-openai:v0.11.0   # >=0.11.0 required for Qwen3.x / Gemma-4 architectures (older images fail with 'architecture not recognized')
   restart: unless-stopped
   networks: [llmao-net]
   volumes:
@@ -48,6 +48,9 @@ services:
       - LLMAO_LITELLM_BASE_URL=http://litellm-proxy:4000
       - LLMAO_LITELLM_MASTER_KEY=${LLMAO_LITELLM_MASTER_KEY}
       - LLMAO_APP_SECRET=${LLMAO_APP_SECRET}
+      - LLMAO_STATE_PATH=/var/opt/llmao-state/llmao-state.json
+    volumes:
+      - llmao-state:/var/opt/llmao-state
     depends_on: [litellm-proxy]
 
   # --- litellm proxy (CPU) --------------------------------------------
@@ -66,7 +69,30 @@ services:
       - LLMAO_SELFHOST_API_KEY=${LLMAO_SELFHOST_API_KEY:-not-required}
       # Admin master key (team/key provisioning + spend).
       - LLMAO_LITELLM_MASTER_KEY=${LLMAO_LITELLM_MASTER_KEY}
-    depends_on: [vllm-gemma, vllm-qwen-coder, vllm-qwen8b]
+      # Postgres backing store — REQUIRED for team/budget/key/spend features
+      # (litellm returns 500 "No db connected" on /team/new without it).
+      - DATABASE_URL=postgresql://litellm:${LLMAO_DB_PASSWORD:-litellm}@litellm-db:5432/litellm
+      - STORE_MODEL_IN_DB=True
+    depends_on: [vllm-gemma, vllm-qwen-coder, vllm-qwen8b, litellm-db]
+
+  # --- litellm backing database (Postgres) ----------------------------
+  # litellm stores teams, virtual keys, budgets, and spend logs here. Without
+  # it, admin endpoints like /team/new fail with 500 "No db connected".
+  litellm-db:
+    image: postgres:16-alpine
+    restart: unless-stopped
+    networks: [llmao-net]
+    environment:
+      - POSTGRES_USER=litellm
+      - POSTGRES_PASSWORD=${LLMAO_DB_PASSWORD:-litellm}
+      - POSTGRES_DB=litellm
+    volumes:
+      - litellm-db-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   # --- vLLM: Gemma 4 26B-A4B (general/multimodal/agentic) --------------
   vllm-gemma:
@@ -109,3 +135,7 @@ networks:
 volumes:
   hf-cache:
     name: llmao-hf-cache
+  litellm-db-data:
+    name: llmao-litellm-db-data
+  llmao-state:
+    name: llmao-state
diff --git a/llmao/catalog.py b/llmao/catalog.py
index a9735e1..620d11a 100644
--- a/llmao/catalog.py
+++ b/llmao/catalog.py
@@ -107,6 +107,7 @@ CATALOG: List[CatalogModel] = [
         api_base_env="LLMAO_SELFHOST_QWEN8B_URL",
         notes="Fast lightweight tier for routine/cheap calls.",
     ),
+
 ]
 
 _BY_ID = {m.id: m for m in CATALOG}
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index c535662..3b4031f 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -70,10 +70,8 @@ def _est_tokens(text: str) -> int:
 
 # Rough per-1k-token USD pricing so the budget math is non-trivial.
 _MOCK_PRICES = {
-    # Self-hosted models have no per-token API bill, but GPU time isn't free.
-    # These nominal per-1k-token prices let budgets/metering exercise the
-    # self-host path (amortized GPU cost). Tune or zero per policy; the real
-    # cost model for self-host is decided by the advisor layer.
+    # Self-hosted: no per-token API bill, but GPU time isn't free. Nominal
+    # prices keep budget/metering exercisable in tests.
     "selfhost/gemma4-26b": 0.0002,
     "selfhost/qwen3.6-27b": 0.0002,
     "selfhost/qwen3-8b": 0.0001,
@@ -273,4 +271,4 @@ class ProxyBackend:
 def make_backend(settings: Settings, store: StateStore) -> Backend:
     if settings.is_mock_llm:
         return MockBackend(settings, store)
-    return ProxyBackend(settings, store)
+    return ProxyBackend(settings, store)
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org