You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/21 12:29:24 UTC
(tooling-llmao) 01/02: deployments: stamp the key on every /model/new, and look ids up by it
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 52818a29da349043028bb92127af5d30b04e936e
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 07:23:50 2026 -0500
deployments: stamp the key on every /model/new, and look ids up by it
Early work item 2 of docs/fleet-reconcile-design.md, the code half.
Every deployment llmao registers now carries model_info.llmao_key, the
canonical URL of its key (the LISTEN port for a box, the config.yaml
base for a provider); the row's model_name is the other half. A LiteLLM
row can then say which of ours it is with no public-to-listen port
translation, and a Vast remap does not move it.
The id lookup behind /model/delete reads the stamp first: a stamped row
is this deployment at whatever address it was registered, so a box that
is down after a remap has its stale deployment deleted instead of "no
LiteLLM id". An unstamped row, from before stamping, is still matched by
its address; that match goes once the backfill has stamped every row.
Presence for the add gate still compares the dialable address, on
purpose: a stamped row at a stale address is not the deployment LiteLLM
needs, and today's path (add at the new address, prune the old as an
orphan) stays until the update action exists.
The hand check of /model/update and the backfill are production steps,
not in this commit.
Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
llmao/proxy/deployments.py | 32 ++++++++++++++++++++---------
tests/test_deployments.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 73 insertions(+), 9 deletions(-)
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index ab8d58d..6bacb43 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -101,7 +101,13 @@ class DeploymentManager:
info: dict[str, Any] = {
"self_hosted": dep.self_hosted,
+ # The dialable base this deployment was registered at, normalised.
"asf_api_base": norm_base(dep.api_base),
+ # The stamp: the deployment's key, so a LiteLLM row says which of
+ # ours it is without translating its public port back to the
+ # listen port. Built from the LISTEN port, so a remap keeps it.
+ # The row's model_name is the other half of the key.
+ "llmao_key": dep.key.url,
}
if dep.access_groups:
# What LiteLLM reads to decide which teams may use this
@@ -133,20 +139,28 @@ class DeploymentManager:
return []
def _litellm_id(self, row: dict, dep) -> str | None:
+ """This deployment's LiteLLM id from one /model/info row, or None when the row is not it.
+
+ The stamp decides when the row has one: a row stamped with this key
+ is this deployment at whatever address it was registered, including
+ a public port that has since been remapped. A row with no stamp is
+ from before stamping, and is matched by its address, as before. The
+ address match goes once the backfill has stamped every row.
+ """
if str(row.get("model_name") or "") != dep.model_name:
return None
- want = norm_base(dep.api_base)
info = row.get("model_info") or {}
if not isinstance(info, dict):
info = {}
- if (
- norm_base(info.get("asf_api_base")) == want
- or norm_base(
- (row.get("litellm_params") or {}).get("api_base") if isinstance(row.get("litellm_params"), dict) else ""
- )
- == want
- ):
- return str(info.get("id") or row.get("model_id") or "") or None
+ row_id = str(info.get("id") or row.get("model_id") or "") or None
+ stamp = info.get("llmao_key")
+ if stamp:
+ return row_id if norm_base(stamp) == dep.key.url else None
+ want = norm_base(dep.api_base)
+ params = row.get("litellm_params")
+ registered = params.get("api_base") if isinstance(params, dict) else ""
+ if want and (norm_base(info.get("asf_api_base")) == want or norm_base(registered) == want):
+ return row_id
return None
def _working_api_key(self, dep) -> str | None:
diff --git a/tests/test_deployments.py b/tests/test_deployments.py
index 98b2d32..64dff3b 100644
--- a/tests/test_deployments.py
+++ b/tests/test_deployments.py
@@ -83,6 +83,7 @@ def test_deployment_body_from_models():
assert body["litellm_params"]["api_base"] == "http://10.0.0.1:8001/v1"
assert body["litellm_params"]["api_key"] == "sk-x"
assert body["model_info"]["asf_api_base"] == "http://10.0.0.1:8001"
+ assert body["model_info"]["llmao_key"] == "http://10.0.0.1:8001"
assert body["model_info"]["self_hosted"] is True
# Our boxes first: LiteLLM uses the lowest order among healthy deployments.
assert body["litellm_params"]["order"] == ORDER_SELF_HOSTED
@@ -299,6 +300,53 @@ def _listen_and_public():
return Fleet(None, [s], models=models)
+def test_the_stamp_is_the_listen_url_not_the_public_one():
+ """A Vast remap changes the public port LiteLLM dials; the stamp does not move."""
+ fleet = _listen_and_public()
+ body = _manager(fleet).deployment_body(fleet.deployments[0])
+ assert body["litellm_params"]["api_base"] == "http://10.0.0.7:12711/v1"
+ assert body["model_info"]["asf_api_base"] == "http://10.0.0.7:12711"
+ assert body["model_info"]["llmao_key"] == "http://10.0.0.7:10100"
+
+
+def test_a_stamped_deployment_is_found_after_a_remap():
+ """The box is down and its deployment points at the old public port.
+
+ Matched by address, the id lookup fails ("no LiteLLM id") and the stale
+ deployment stays. Matched by the stamp, it is deleted.
+ """
+ fleet = _listen_and_public()
+ (dep,) = fleet.deployments
+ dep.vllm.state = UNHEALTHY
+ dep.in_litellm = True
+ stale = {
+ "model_name": "gemma4-26b",
+ "litellm_params": {"api_base": "http://10.0.0.7:11111/v1"},
+ "model_info": {"id": "old", "asf_api_base": "http://10.0.0.7:11111", "llmao_key": "http://10.0.0.7:10100"},
+ }
+ be = _manager(fleet, _litellm([stale]))
+ asyncio.run(be.sync_selfhost())
+ sent = [c.json["id"] for c in be._transport.calls if c.path == "model/delete"]
+ assert sent == ["old"]
+ assert dep.in_litellm is False
+
+
+def test_a_stamp_for_another_key_is_not_this_deployment():
+ """Same model name and, by coincidence, the same address: the stamp says it is another box's."""
+ fleet = _listen_and_public()
+ (dep,) = fleet.deployments
+ be = _manager(fleet)
+ row = {
+ "model_name": "gemma4-26b",
+ "litellm_params": {"api_base": "http://10.0.0.7:12711/v1"},
+ "model_info": {"id": "theirs", "llmao_key": "http://10.0.0.8:8001"},
+ }
+ assert be._litellm_id(row, dep) is None
+ # Unstamped, the address still decides, until the backfill.
+ del row["model_info"]["llmao_key"]
+ assert be._litellm_id(row, dep) == "theirs"
+
+
def test_sync_selfhost_does_not_repost_the_same_deployment():
fleet = _listen_and_public()
be = _manager(fleet, _info("10.0.0.7", 12711))
@@ -387,6 +435,8 @@ def test_a_provider_deployment_is_built_from_config():
assert dep.access_groups == set()
assert "access_groups" not in body["model_info"]
assert ORDER_SPEND > ORDER_SELF_HOSTED
+ # Stamped like any other deployment: the provider base, and the row's model_name.
+ assert body["model_info"]["llmao_key"] == "https://api.x.ai"
def test_ensure_providers_registers_each_one_once():
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org