You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/08/05 17:05:18 UTC
(tooling-llmao) 07/17: Remove upstream providers from the catalog (self-host only)
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 5281fc6b31e3297ae667c9350690437ca3609d70
Author: Andrew Musselman <an...@gmail.com>
AuthorDate: Tue Jul 7 13:36:46 2026 -0700
Remove upstream providers from the catalog (self-host only)
Drop the Anthropic and Bedrock entries so the PoC catalog lists only the
three self-hosted vLLM models. With the providers listed but their keys
unset, those models showed up in the portal and failed only when selected;
removing them keeps the portal to what actually works.
- llmao/catalog.py: remove the anthropic and bedrock CatalogModel entries;
CATALOG is now gemma4-26b, qwen3.6-27b, qwen3-8b.
- litellm/config.yaml: regenerated from the catalog (no external blocks).
- llmao/litellm_client.py: mock price map keyed to the self-host backends.
Priced nominally (not zero) so budget enforcement stays exercisable in
tests; real self-host cost accounting is the advisor layer's job.
- tests/test_phase1.py: use self-host/gemma4-26b as the test model in place
of the removed anthropic/claude-haiku.
- infra/docker/docker-compose.yml: drop the ANTHROPIC and BEDROCK env
passthroughs.
The renderer PROVIDER_ENV map is left in place (dormant) so a provider can
be reinstated later by adding a catalog entry, without renderer changes.
---
infra/docker/docker-compose.yml | 3 ---
litellm/config.yaml | 8 --------
llmao/catalog.py | 29 -----------------------------
llmao/litellm_client.py | 14 ++++++++------
tests/test_phase1.py | 14 +++++++-------
5 files changed, 15 insertions(+), 53 deletions(-)
diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml
index f83b59e..55b540e 100644
--- a/infra/docker/docker-compose.yml
+++ b/infra/docker/docker-compose.yml
@@ -64,9 +64,6 @@ services:
- LLMAO_SELFHOST_QWEN_CODER_URL=http://vllm-qwen-coder:8002/v1
- LLMAO_SELFHOST_QWEN8B_URL=http://vllm-qwen8b:8003/v1
- LLMAO_SELFHOST_API_KEY=${LLMAO_SELFHOST_API_KEY:-not-required}
- # External providers.
- - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
- - AWS_BEARER_TOKEN_BEDROCK=${AWS_BEARER_TOKEN_BEDROCK}
# Admin master key (team/key provisioning + spend).
- LLMAO_LITELLM_MASTER_KEY=${LLMAO_LITELLM_MASTER_KEY}
depends_on: [vllm-gemma, vllm-qwen-coder, vllm-qwen8b]
diff --git a/litellm/config.yaml b/litellm/config.yaml
index 9f7f81d..a7fda89 100644
--- a/litellm/config.yaml
+++ b/litellm/config.yaml
@@ -17,14 +17,6 @@ model_list:
model: openai/qwen3-8b
api_base: os.environ/LLMAO_SELFHOST_QWEN8B_URL
api_key: os.environ/LLMAO_SELFHOST_API_KEY
- - model_name: anthropic/claude-haiku-4-5
- litellm_params:
- model: anthropic/claude-haiku-4-5
- api_key: os.environ/ANTHROPIC_API_KEY
- - model_name: bedrock/anthropic.claude-sonnet-4-5
- litellm_params:
- model: bedrock/anthropic.claude-sonnet-4-5
- api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK
litellm_settings:
drop_params: true
diff --git a/llmao/catalog.py b/llmao/catalog.py
index 9cc91f2..a9735e1 100644
--- a/llmao/catalog.py
+++ b/llmao/catalog.py
@@ -107,35 +107,6 @@ CATALOG: List[CatalogModel] = [
api_base_env="LLMAO_SELFHOST_QWEN8B_URL",
notes="Fast lightweight tier for routine/cheap calls.",
),
-
- # --- external providers ---------------------------------------------
- CatalogModel(
- id="anthropic/claude-haiku",
- display_name="Claude Haiku",
- provider="anthropic",
- backend="anthropic/claude-haiku-4-5",
- context_window=200000,
- license="proprietary",
- openness="proprietary",
- weights_distribution="n/a (API only)",
- training_data_provenance="undisclosed (Anthropic)",
- provenance_record="absent",
- notes="External provider.",
- ),
- CatalogModel(
- id="bedrock/claude-sonnet",
- display_name="Claude Sonnet (Bedrock)",
- provider="bedrock",
- backend="bedrock/anthropic.claude-sonnet-4-5",
- context_window=200000,
- license="proprietary",
- openness="proprietary",
- weights_distribution="n/a (API only, via AWS Bedrock)",
- training_data_provenance="undisclosed (Anthropic)",
- provenance_record="absent",
- notes="External provider via AWS Bedrock — the safer data layer for "
- "sensitive content. Funds the security pipeline today.",
- ),
]
_BY_ID = {m.id: m for m in CATALOG}
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index b3758a0..c535662 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -70,11 +70,13 @@ def _est_tokens(text: str) -> int:
# Rough per-1k-token USD pricing so the budget math is non-trivial.
_MOCK_PRICES = {
- "openai/llama-3.1-8b-instruct": 0.0, # self-hosted: no marginal API cost
- "openai/mistral-7b-instruct": 0.0,
- "openai/gpt-4o-mini": 0.0006,
- "anthropic/claude-haiku-4-5": 0.0010,
- "anthropic/claude-haiku": 0.0010,
+ # Self-hosted models have no per-token API bill, but GPU time isn't free.
+ # These nominal per-1k-token prices let budgets/metering exercise the
+ # self-host path (amortized GPU cost). Tune or zero per policy; the real
+ # cost model for self-host is decided by the advisor layer.
+ "selfhost/gemma4-26b": 0.0002,
+ "selfhost/qwen3.6-27b": 0.0002,
+ "selfhost/qwen3-8b": 0.0001,
}
@@ -271,4 +273,4 @@ class ProxyBackend:
def make_backend(settings: Settings, store: StateStore) -> Backend:
if settings.is_mock_llm:
return MockBackend(settings, store)
- return ProxyBackend(settings, store)
\ No newline at end of file
+ return ProxyBackend(settings, store)
diff --git a/tests/test_phase1.py b/tests/test_phase1.py
index 5a6b1d9..4bea6d2 100644
--- a/tests/test_phase1.py
+++ b/tests/test_phase1.py
@@ -43,7 +43,7 @@ def test_member_can_call_and_is_metered():
with tempfile.TemporaryDirectory() as tmp:
seam, _ = _seam(tmp)
ident = Identity(uid="jdoe", projects=["airflow"], committees=[])
- c = seam.chat(ident, "airflow", "anthropic/claude-haiku", [{"role": "user", "content": "hi there"}])
+ c = seam.chat(ident, "airflow", "self-host/gemma4-26b", [{"role": "user", "content": "hi there"}])
assert c.content
assert c.prompt_tokens > 0
info = seam.team_status("airflow")
@@ -56,7 +56,7 @@ def test_non_member_is_refused():
seam, _ = _seam(tmp)
ident = Identity(uid="jdoe", projects=["airflow"], committees=[])
with pytest.raises(AuthzError):
- seam.chat(ident, "kafka", "anthropic/claude-haiku", [{"role": "user", "content": "hi"}])
+ seam.chat(ident, "kafka", "self-host/gemma4-26b", [{"role": "user", "content": "hi"}])
def test_unknown_model_is_refused():
@@ -75,7 +75,7 @@ def test_budget_is_enforced():
tripped = False
for _ in range(500):
try:
- seam.chat(ident, "airflow", "anthropic/claude-haiku",
+ seam.chat(ident, "airflow", "self-host/gemma4-26b",
[{"role": "user", "content": "x" * 4000}])
except BudgetExceeded:
tripped = True
@@ -88,7 +88,7 @@ def test_activity_requires_pmc_admin():
seam, _ = _seam(tmp)
member = Identity(uid="jdoe", projects=["airflow"], committees=[])
admin = Identity(uid="chair", projects=[], committees=["airflow"])
- seam.chat(member, "airflow", "anthropic/claude-haiku", [{"role": "user", "content": "hi"}])
+ seam.chat(member, "airflow", "self-host/gemma4-26b", [{"role": "user", "content": "hi"}])
with pytest.raises(AuthzError):
seam.project_activity(member, "airflow")
rows = seam.project_activity(admin, "airflow")
@@ -100,7 +100,7 @@ def test_site_admin_sees_any_project():
seam, _ = _seam(tmp)
member = Identity(uid="jdoe", projects=["airflow"], committees=[])
root = Identity(uid="root", projects=[], committees=[], is_site_admin=True)
- seam.chat(member, "airflow", "anthropic/claude-haiku", [{"role": "user", "content": "hi"}])
+ seam.chat(member, "airflow", "self-host/gemma4-26b", [{"role": "user", "content": "hi"}])
rows = seam.project_activity(root, "airflow")
assert len(rows) == 1
@@ -122,7 +122,7 @@ def test_http_chat_flow():
assert r.status_code == 200
# chat
r = await client.post("/v1/chat/completions", json={
- "model": "anthropic/claude-haiku",
+ "model": "self-host/gemma4-26b",
"messages": [{"role": "user", "content": "hello"}],
}, headers={"X-LLMAO-Project": "airflow"})
assert r.status_code == 200, await r.get_data()
@@ -146,7 +146,7 @@ def test_http_requires_auth():
async def run():
client = app.test_client()
r = await client.post("/v1/chat/completions", json={
- "model": "anthropic/claude-haiku",
+ "model": "self-host/gemma4-26b",
"messages": [{"role": "user", "content": "hi"}],
})
assert r.status_code == 401
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org