You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/21 12:29:23 UTC
(tooling-llmao) branch main updated (7bd46bc -> 08815b4)
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
from 7bd46bc deployments: re-read the LiteLLM deployments after every write
new 52818a2 deployments: stamp the key on every /model/new, and look ids up by it
new 08815b4 reconcile: the key's display form, used by every log line and page cell
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
llmao/fleet.py | 4 ++--
llmao/proxy/deployments.py | 46 ++++++++++++++++++++++++++--------------
llmao/reconcile.py | 14 +++++++++++++
pages.py | 9 ++++----
tests/test_deployments.py | 52 +++++++++++++++++++++++++++++++++++++++++++++-
tests/test_reconcile.py | 11 ++++++++++
6 files changed, 112 insertions(+), 24 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-llmao) 01/02: deployments: stamp the key on every /model/new, and look ids up by it
Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 52818a29da349043028bb92127af5d30b04e936e
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 07:23:50 2026 -0500
deployments: stamp the key on every /model/new, and look ids up by it
Early work item 2 of docs/fleet-reconcile-design.md, the code half.
Every deployment llmao registers now carries model_info.llmao_key, the
canonical URL of its key (the LISTEN port for a box, the config.yaml
base for a provider); the row's model_name is the other half. A LiteLLM
row can then say which of ours it is with no public-to-listen port
translation, and a Vast remap does not move it.
The id lookup behind /model/delete reads the stamp first: a stamped row
is this deployment at whatever address it was registered, so a box that
is down after a remap has its stale deployment deleted instead of "no
LiteLLM id". An unstamped row, from before stamping, is still matched by
its address; that match goes once the backfill has stamped every row.
Presence for the add gate still compares the dialable address, on
purpose: a stamped row at a stale address is not the deployment LiteLLM
needs, and today's path (add at the new address, prune the old as an
orphan) stays until the update action exists.
The hand check of /model/update and the backfill are production steps,
not in this commit.
Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
llmao/proxy/deployments.py | 32 ++++++++++++++++++++---------
tests/test_deployments.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 73 insertions(+), 9 deletions(-)
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index ab8d58d..6bacb43 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -101,7 +101,13 @@ class DeploymentManager:
info: dict[str, Any] = {
"self_hosted": dep.self_hosted,
+ # The dialable base this deployment was registered at, normalised.
"asf_api_base": norm_base(dep.api_base),
+ # The stamp: the deployment's key, so a LiteLLM row says which of
+ # ours it is without translating its public port back to the
+ # listen port. Built from the LISTEN port, so a remap keeps it.
+ # The row's model_name is the other half of the key.
+ "llmao_key": dep.key.url,
}
if dep.access_groups:
# What LiteLLM reads to decide which teams may use this
@@ -133,20 +139,28 @@ class DeploymentManager:
return []
def _litellm_id(self, row: dict, dep) -> str | None:
+ """This deployment's LiteLLM id from one /model/info row, or None when the row is not it.
+
+ The stamp decides when the row has one: a row stamped with this key
+ is this deployment at whatever address it was registered, including
+ a public port that has since been remapped. A row with no stamp is
+ from before stamping, and is matched by its address, as before. The
+ address match goes once the backfill has stamped every row.
+ """
if str(row.get("model_name") or "") != dep.model_name:
return None
- want = norm_base(dep.api_base)
info = row.get("model_info") or {}
if not isinstance(info, dict):
info = {}
- if (
- norm_base(info.get("asf_api_base")) == want
- or norm_base(
- (row.get("litellm_params") or {}).get("api_base") if isinstance(row.get("litellm_params"), dict) else ""
- )
- == want
- ):
- return str(info.get("id") or row.get("model_id") or "") or None
+ row_id = str(info.get("id") or row.get("model_id") or "") or None
+ stamp = info.get("llmao_key")
+ if stamp:
+ return row_id if norm_base(stamp) == dep.key.url else None
+ want = norm_base(dep.api_base)
+ params = row.get("litellm_params")
+ registered = params.get("api_base") if isinstance(params, dict) else ""
+ if want and (norm_base(info.get("asf_api_base")) == want or norm_base(registered) == want):
+ return row_id
return None
def _working_api_key(self, dep) -> str | None:
diff --git a/tests/test_deployments.py b/tests/test_deployments.py
index 98b2d32..64dff3b 100644
--- a/tests/test_deployments.py
+++ b/tests/test_deployments.py
@@ -83,6 +83,7 @@ def test_deployment_body_from_models():
assert body["litellm_params"]["api_base"] == "http://10.0.0.1:8001/v1"
assert body["litellm_params"]["api_key"] == "sk-x"
assert body["model_info"]["asf_api_base"] == "http://10.0.0.1:8001"
+ assert body["model_info"]["llmao_key"] == "http://10.0.0.1:8001"
assert body["model_info"]["self_hosted"] is True
# Our boxes first: LiteLLM uses the lowest order among healthy deployments.
assert body["litellm_params"]["order"] == ORDER_SELF_HOSTED
@@ -299,6 +300,53 @@ def _listen_and_public():
return Fleet(None, [s], models=models)
+def test_the_stamp_is_the_listen_url_not_the_public_one():
+ """A Vast remap changes the public port LiteLLM dials; the stamp does not move."""
+ fleet = _listen_and_public()
+ body = _manager(fleet).deployment_body(fleet.deployments[0])
+ assert body["litellm_params"]["api_base"] == "http://10.0.0.7:12711/v1"
+ assert body["model_info"]["asf_api_base"] == "http://10.0.0.7:12711"
+ assert body["model_info"]["llmao_key"] == "http://10.0.0.7:10100"
+
+
+def test_a_stamped_deployment_is_found_after_a_remap():
+ """The box is down and its deployment points at the old public port.
+
+ Matched by address, the id lookup fails ("no LiteLLM id") and the stale
+ deployment stays. Matched by the stamp, it is deleted.
+ """
+ fleet = _listen_and_public()
+ (dep,) = fleet.deployments
+ dep.vllm.state = UNHEALTHY
+ dep.in_litellm = True
+ stale = {
+ "model_name": "gemma4-26b",
+ "litellm_params": {"api_base": "http://10.0.0.7:11111/v1"},
+ "model_info": {"id": "old", "asf_api_base": "http://10.0.0.7:11111", "llmao_key": "http://10.0.0.7:10100"},
+ }
+ be = _manager(fleet, _litellm([stale]))
+ asyncio.run(be.sync_selfhost())
+ sent = [c.json["id"] for c in be._transport.calls if c.path == "model/delete"]
+ assert sent == ["old"]
+ assert dep.in_litellm is False
+
+
+def test_a_stamp_for_another_key_is_not_this_deployment():
+ """Same model name and, by coincidence, the same address: the stamp says it is another box's."""
+ fleet = _listen_and_public()
+ (dep,) = fleet.deployments
+ be = _manager(fleet)
+ row = {
+ "model_name": "gemma4-26b",
+ "litellm_params": {"api_base": "http://10.0.0.7:12711/v1"},
+ "model_info": {"id": "theirs", "llmao_key": "http://10.0.0.8:8001"},
+ }
+ assert be._litellm_id(row, dep) is None
+ # Unstamped, the address still decides, until the backfill.
+ del row["model_info"]["llmao_key"]
+ assert be._litellm_id(row, dep) == "theirs"
+
+
def test_sync_selfhost_does_not_repost_the_same_deployment():
fleet = _listen_and_public()
be = _manager(fleet, _info("10.0.0.7", 12711))
@@ -387,6 +435,8 @@ def test_a_provider_deployment_is_built_from_config():
assert dep.access_groups == set()
assert "access_groups" not in body["model_info"]
assert ORDER_SPEND > ORDER_SELF_HOSTED
+ # Stamped like any other deployment: the provider base, and the row's model_name.
+ assert body["model_info"]["llmao_key"] == "https://api.x.ai"
def test_ensure_providers_registers_each_one_once():
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-llmao) 02/02: reconcile: the key's display form, used by every log line and page cell
Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 08815b4c15308e5be3c5ea6a93272e94f35d3337
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 07:26:59 2026 -0500
reconcile: the key's display form, used by every log line and page cell
DeploymentKey.display is host:port for a box (the listen port) and the
base for a provider, never a dialable URL: the public port is a fact
about reaching the box, not about which deployment it is, and a URL on
a page gets cut and pasted. The deployment manager's and the server's
log lines, and the ident the /models page prints, now come from it, so
a Vast remap no longer changes how a deployment is named anywhere.
Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
llmao/fleet.py | 4 ++--
llmao/proxy/deployments.py | 14 +++++++-------
llmao/reconcile.py | 14 ++++++++++++++
pages.py | 9 ++++-----
tests/test_deployments.py | 2 +-
tests/test_reconcile.py | 11 +++++++++++
6 files changed, 39 insertions(+), 15 deletions(-)
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 1612f63..0763822 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -612,9 +612,9 @@ class VllmServer:
)
self.state = self_hosted_lifecycle(snap, cfg, now)
if self.state == HEALTHY and was != HEALTHY:
- _LOGGER.info("fleet server %s@%s healthy", self.name, self.api_base)
+ _LOGGER.info("fleet server %s@%s healthy", self.name, self.key.display)
elif self.state in (UNHEALTHY, STALLED) and was not in (UNHEALTHY, STALLED):
- _LOGGER.warning("fleet server %s@%s %s (%s)", self.name, self.api_base, self.state, self.last_error)
+ _LOGGER.warning("fleet server %s@%s %s (%s)", self.name, self.key.display, self.state, self.last_error)
# What a deployment's cost figure means, when it is not simply money owed.
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index 6bacb43..85d79d2 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -190,17 +190,17 @@ class DeploymentManager:
"""
if await self._deployment_present(dep):
dep.in_litellm = True
- _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
+ _LOGGER.info("model/new already present %s@%s", dep.name, dep.key.display)
return False
body = self.deployment_body(dep, api_key=self._working_api_key(dep))
resp = await self._transport.request("POST", "model/new", json=body)
if resp.status_code in (400, 409):
dep.in_litellm = True
- _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
+ _LOGGER.info("model/new already present %s@%s", dep.name, dep.key.display)
return True
self._transport.raise_http(resp)
dep.in_litellm = True
- _LOGGER.info("model/new %s@%s", dep.name, dep.api_base)
+ _LOGGER.info("model/new %s@%s", dep.name, dep.key.display)
return True
async def delete_deployment(self, dep) -> bool:
@@ -214,12 +214,12 @@ class DeploymentManager:
break
if not found:
dep.in_litellm = False
- _LOGGER.warning("model/delete: no LiteLLM id for %s@%s", dep.name, dep.api_base)
+ _LOGGER.warning("model/delete: no LiteLLM id for %s@%s", dep.name, dep.key.display)
return False
resp = await self._transport.request("POST", "model/delete", json={"id": found})
self._transport.raise_http(resp)
dep.in_litellm = False
- _LOGGER.info("model/delete %s@%s id=%s", dep.name, dep.api_base, found)
+ _LOGGER.info("model/delete %s@%s id=%s", dep.name, dep.key.display, found)
return True
async def _each(self, action, dep) -> bool:
@@ -233,7 +233,7 @@ class DeploymentManager:
try:
wrote = await action(dep)
except Exception as e:
- _LOGGER.exception("%s failed for %s@%s", action.__name__, dep.name, dep.api_base)
+ _LOGGER.exception("%s failed for %s@%s", action.__name__, dep.name, dep.key.display)
# Kept for /models, which otherwise shows only that it is missing.
dep.litellm_error = str(e) or type(e).__name__
dep.litellm_error_at = time.time()
@@ -497,7 +497,7 @@ class DeploymentManager:
fresh = [badge for badge in view.skew if badge not in dep.skew]
dep.skew = list(view.skew)
for badge in fresh:
- _LOGGER.warning("skew: %s@%s %s", dep.name, dep.api_base, SKEW_PHRASE[badge])
+ _LOGGER.warning("skew: %s@%s %s", dep.name, dep.key.display, SKEW_PHRASE[badge])
def _index_deployments(rows) -> tuple[dict[str, list[str]], set[tuple[str, str]]]:
diff --git a/llmao/reconcile.py b/llmao/reconcile.py
index 1a12e24..edb7ee3 100644
--- a/llmao/reconcile.py
+++ b/llmao/reconcile.py
@@ -90,6 +90,20 @@ class DeploymentKey(NamedTuple):
url: str
model_name: str
+ @property
+ def display(self) -> str:
+ """How a page or a log line names the deployment: host:port for a box, the base for a provider.
+
+ Never a dialable URL. The public port is a fact about reaching the
+ box, not about which deployment it is, and a URL on a page gets cut
+ and pasted. A box's key is its listen URL, so this is host and listen
+ port. A provider's key is its base, which is already the name to use.
+ """
+ parsed = urlparse(self.url)
+ if parsed.port and parsed.path in ("", "/"):
+ return parsed.netloc
+ return self.url
+
def deployment_key(
model_name: str,
diff --git a/pages.py b/pages.py
index d3788cd..ee97082 100644
--- a/pages.py
+++ b/pages.py
@@ -470,11 +470,10 @@ def _badge_class(state: str, *, routed: bool = True) -> str:
def _dep_ident(dep) -> str:
- """host:port for a box (the listen port from fleet.hosts), else the endpoint."""
- srv = dep.vllm
- if srv is not None:
- return f"{srv.host}:{srv.listen_port}"
- return dep.api_base or dep.name
+ """The key's display form: host:port for a box (the listen port), the base for a provider."""
+ if dep.vllm is not None or dep.api_base:
+ return dep.key.display
+ return dep.name
def _dep_where(dep) -> edict:
diff --git a/tests/test_deployments.py b/tests/test_deployments.py
index 64dff3b..62e09d7 100644
--- a/tests/test_deployments.py
+++ b/tests/test_deployments.py
@@ -171,7 +171,7 @@ def test_one_failing_deployment_does_not_stop_the_rest(caplog):
assert posted == ["http://10.0.0.1:8001/v1", "http://10.0.0.2:8001/v1"]
assert [d.in_litellm for d in fleet.deployments] == [False, True, True]
# Says which deployment, so the log points at the cause.
- assert "add_deployment failed for gone@http://10.0.0.9:8001/v1" in caplog.text
+ assert "add_deployment failed for gone@10.0.0.9:8001" in caplog.text
def test_sync_selfhost_deletes_when_down():
diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py
index df1be38..465ea52 100644
--- a/tests/test_reconcile.py
+++ b/tests/test_reconcile.py
@@ -123,6 +123,17 @@ def test_same_name_from_two_providers_are_two_keys():
)
+def test_display_is_host_and_listen_port_for_a_box():
+ """What pages and logs print. Never the public port, never a URL to paste."""
+ assert deployment_key("gemma4-26b", host="192.0.2.10", listen_port=8001).display == "192.0.2.10:8001"
+ assert deployment_key("gemma4-26b", host="2001:db8::1", listen_port=8001).display == "[2001:db8::1]:8001"
+
+
+def test_display_is_the_base_for_a_provider():
+ assert deployment_key("grok-4", api_base="https://api.x.ai/v1").display == "https://api.x.ai"
+ assert deployment_key("grok-4", api_base="https://openrouter.ai/api/v1").display == "https://openrouter.ai/api"
+
+
def test_key_refuses_half_an_identity():
with pytest.raises(ValueError):
deployment_key("", host="192.0.2.10", listen_port=8001)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org