You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/09/08 23:28:45 UTC
(tooling-llmao) branch main updated: normalise api_base, and stop requiring api_key in the catalog
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
The following commit(s) were added to refs/heads/main by this push:
new 47eb9a5 normalise api_base, and stop requiring api_key in the catalog
47eb9a5 is described below
commit 47eb9a5e6eb193b970b154fd55f45c0d28d27f5c
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Tue Sep 8 16:28:12 2026 -0700
normalise api_base, and stop requiring api_key in the catalog
LiteLLM appends /v1/chat/completions to api_base, so a base already ending
in /v1 resolves to /v1/v1/... That 404s at the origin, and LiteLLM reports
it as an OpenAI authentication error naming platform.openai.com -- which
sends the reader after their own key rather than the route. Cost real
debugging time against the Fastly-fronted Gemma endpoint.
_norm_base already stripped trailing slashes; it now strips a trailing /v1
as well. This also fixes the skew comparison: Server.api_base is host:port
with no suffix, so a catalog entry carrying /v1 never matched and the check
reported a route as missing when it was present.
The catalog example drops api_base and api_key from litellm_params. Both
are per-instance -- where to serve, and the credential for that one box --
rather than properties of the model. validate_fleet and Server.from_row are
relaxed to match. api_key is now empty until the health-gated registration
work generates one per instance, so a box provisioned from this will start
vLLM without a key; do not deploy from it until that lands.
Adds tests for the /v1 cases, including the skew comparison.
Closes #4. The stopgap it proposed is not needed -- putting api_base back
into model_list.yaml would mix "how to serve" with "where to serve", which
is the concern gstein raised on that issue. The /v1 gotcha it documented is
fixed here rather than lost with it.
---
llmao/fleet.py | 9 ++++++---
llmao/litellm_client.py | 12 +++++++++++-
tests/test_fleet.py | 35 +++++++++++++++++++++++++++++++++++
3 files changed, 52 insertions(+), 4 deletions(-)
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 8cdf769..3e5e07c 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -105,8 +105,8 @@ def validate_fleet(cfg: Any, models: list | None = None) -> None:
vllm = model.model_info.vllm
if "model" not in vllm or not vllm.model:
raise ValueError(f"{name}: model_info.vllm.model is required")
- if "litellm_params" not in model or not model.litellm_params.api_key:
- raise ValueError(f"{name}: litellm_params.api_key is required")
+ if "litellm_params" not in model:
+ raise ValueError(f"{name}: litellm_params is required")
catalog = set(names)
for host, rows in hosts.items():
@@ -204,7 +204,10 @@ class Server:
host=host,
port=port,
hf_model=str(vllm.model),
- api_key=str(model.litellm_params.api_key),
+ # Per-instance credential, not a catalog property -- generated when
+ # a host is added, same reasoning as api_base. Empty until the
+ # health-gated registration work lands.
+ api_key=str(model.litellm_params.get("api_key") or ""),
args=[str(a) for a in args],
gpu_memory_utilization=float(util) if util is not None else None,
max_model_len=int(maxlen) if maxlen is not None else None,
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index 45fb9bf..cfca59c 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -533,7 +533,17 @@ class LiteLLMBackend:
def _norm_base(url: Any) -> str:
- return str(url or "").rstrip("/")
+ """Strip trailing slashes and a trailing /v1.
+
+ LiteLLM appends /v1/chat/completions to api_base, so a base ending in
+ /v1 resolves to /v1/v1/... That 404s at the origin and surfaces as an
+ OpenAI authentication error naming platform.openai.com, which sends the
+ reader after their own key rather than the route.
+ """
+ s = str(url or "").rstrip("/")
+ if s.endswith("/v1"):
+ s = s[:-3].rstrip("/")
+ return s
def _api_bases_from_model_info(body: Any) -> set[str]:
diff --git a/tests/test_fleet.py b/tests/test_fleet.py
index b8f2f10..9643a5a 100644
--- a/tests/test_fleet.py
+++ b/tests/test_fleet.py
@@ -13,6 +13,7 @@ from llmao.fleet import (
validate_fleet,
)
from llmao.models import load_model_list
+from llmao.litellm_client import _norm_base
EXAMPLE = Path(__file__).resolve().parent.parent / "model_list.yaml.example"
EXAMPLE_CFG = Path(__file__).resolve().parent.parent / "config.yaml.example"
@@ -112,3 +113,37 @@ def test_client_ip_xff_leftmost():
def test_client_ip_no_xff():
assert client_ip(remote_addr="203.0.113.10", forwarded_for=None) == "203.0.113.10"
assert client_ip(remote_addr="::ffff:203.0.113.10", forwarded_for="") == "203.0.113.10"
+
+
+def test_norm_base_strips_trailing_v1():
+ """LiteLLM appends /v1/chat/completions to api_base.
+
+ A base that already ends in /v1 resolves to /v1/v1/..., which 404s at the
+ origin. LiteLLM cannot parse that and reports an OpenAI authentication
+ error naming platform.openai.com -- so the symptom points at the caller's
+ key rather than at the route. This happened in production against the
+ Fastly-fronted Gemma endpoint.
+ """
+ assert _norm_base("https://llm.tooling.apache.org/v1") == "https://llm.tooling.apache.org"
+ assert _norm_base("https://llm.tooling.apache.org/v1/") == "https://llm.tooling.apache.org"
+ assert _norm_base("http://127.0.0.1:8003/v1") == "http://127.0.0.1:8003"
+
+
+def test_norm_base_leaves_other_paths_alone():
+ """Only a trailing /v1 is stripped -- not a path that merely contains it."""
+ assert _norm_base("http://127.0.0.1:8003") == "http://127.0.0.1:8003"
+ assert _norm_base("http://127.0.0.1:8003/") == "http://127.0.0.1:8003"
+ assert _norm_base("https://example.com/v1/proxy") == "https://example.com/v1/proxy"
+ assert _norm_base("") == ""
+ assert _norm_base(None) == ""
+
+
+def test_norm_base_makes_skew_comparison_match():
+ """The comparison this exists for.
+
+ check_config_skew compares Server.api_base against what LiteLLM reports.
+ Server.api_base is host:port with no suffix; a catalog entry may carry
+ /v1. Without normalisation on both sides they never match, and the skew
+ check reports a missing route that is actually present.
+ """
+ assert _norm_base("http://100.105.28.100:8003/v1") == _norm_base("http://100.105.28.100:8003")
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org