You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/21 11:48:25 UTC

(tooling-llmao) branch main updated (62c00db -> 7bd46bc)

This is an automated email from the ASF dual-hosted git repository.

gstein pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git


    from 62c00db  docs: fleet reconcile, the not-reached grace clock and the listen URL fallback
     new 9fbee0e  reconcile: the deployment key, frozen with tests
     new b9b5ce3  fleet: ORPHAN_PASSES for the skew clock, UNLISTED_PASSES for the listings
     new 039c403  fleet: one working-key rule, shared by the deployment body and /v1/models
     new 7bd46bc  deployments: re-read the LiteLLM deployments after every write

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 llmao/fleet.py             |  68 +++++++++++----
 llmao/proxy/deployments.py | 202 ++++++++++++++++++++++++---------------------
 llmao/reconcile.py         | 123 +++++++++++++++++++++++++++
 tests/test_deployments.py  |  84 +++++++++++++++++--
 tests/test_fleet.py        |  97 +++++++++++-----------
 tests/test_reconcile.py    | 136 ++++++++++++++++++++++++++++++
 6 files changed, 543 insertions(+), 167 deletions(-)
 create mode 100644 llmao/reconcile.py
 create mode 100644 tests/test_reconcile.py


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-llmao) 04/04: deployments: re-read the LiteLLM deployments after every write

Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git

commit 7bd46bc7be17ed32e1dcf5ca261cf287f3e8b227
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 06:45:48 2026 -0500

    deployments: re-read the LiteLLM deployments after every write
    
    Early work item 4 of docs/fleet-reconcile-design.md. add_deployment and
    delete_deployment set in_litellm on the strength of LiteLLM's 200. Now
    each reports whether it sent a write, and sync_selfhost and the orphan
    prune follow any write with one GET /model/info, so presence is observed
    and not assumed. LiteLLM 1.105.0 updates its router inside the model/new
    and model/delete handlers before answering, so that read sees the write.
    If the read fails, the last answer stands and the next skew pass reads
    again; the probe tick never raises because of it.
    
    The presence rule check_config_skew applied per deployment is factored
    into _mark_presence, which the confirm read shares. Tests get a stand-in
    LiteLLM that remembers its deployments, so a read after a write sees it.
    
    Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
 llmao/proxy/deployments.py | 131 ++++++++++++++++++++++++++++++---------------
 tests/test_deployments.py  |  76 ++++++++++++++++++++++++--
 2 files changed, 159 insertions(+), 48 deletions(-)

diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index e84850c..ab8d58d 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -166,8 +166,8 @@ class DeploymentManager:
             return False
         return any(dep.vllm.same_deployment(host, port, model_name) for _norm, host, port, model_name, _id in routes)
 
-    async def add_deployment(self, dep) -> None:
-        """POST /model/new for this deployment. No serving check.
+    async def add_deployment(self, dep) -> bool:
+        """POST /model/new for this deployment. No serving check. True when a write was sent.
 
         Callers decide the gate. sync_selfhost waits until vLLM is SERVING.
         POST /do-add-deployment uses add_refusal for the same gate, then calls
@@ -177,18 +177,20 @@ class DeploymentManager:
         if await self._deployment_present(dep):
             dep.in_litellm = True
             _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
-            return
+            return False
         body = self.deployment_body(dep, api_key=self._working_api_key(dep))
         resp = await self._transport.request("POST", "model/new", json=body)
         if resp.status_code in (400, 409):
             dep.in_litellm = True
             _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
-            return
+            return True
         self._transport.raise_http(resp)
         dep.in_litellm = True
         _LOGGER.info("model/new %s@%s", dep.name, dep.api_base)
+        return True
 
-    async def delete_deployment(self, dep) -> None:
+    async def delete_deployment(self, dep) -> bool:
+        """POST /model/delete for this deployment's LiteLLM id. True when a write was sent."""
         rows = await self._model_info_rows()
         found = None
         for row in rows:
@@ -199,29 +201,51 @@ class DeploymentManager:
         if not found:
             dep.in_litellm = False
             _LOGGER.warning("model/delete: no LiteLLM id for %s@%s", dep.name, dep.api_base)
-            return
+            return False
         resp = await self._transport.request("POST", "model/delete", json={"id": found})
         self._transport.raise_http(resp)
         dep.in_litellm = False
         _LOGGER.info("model/delete %s@%s id=%s", dep.name, dep.api_base, found)
+        return True
 
-    async def _each(self, action, dep) -> None:
+    async def _each(self, action, dep) -> bool:
         """Run `action` on one deployment; a failure is logged and goes no further.
 
         The callers walk every deployment. Unguarded, the first one to raise
         ended the walk, and every deployment after it was never registered:
         vLLM up and Healthy, nothing in LiteLLM, the same on every pass.
+        Returns whether the action sent a write to LiteLLM.
         """
         try:
-            await action(dep)
+            wrote = await action(dep)
         except Exception as e:
             _LOGGER.exception("%s failed for %s@%s", action.__name__, dep.name, dep.api_base)
             # Kept for /models, which otherwise shows only that it is missing.
             dep.litellm_error = str(e) or type(e).__name__
             dep.litellm_error_at = time.time()
-        else:
-            dep.litellm_error = ""
-            dep.litellm_error_at = None
+            return False
+        dep.litellm_error = ""
+        dep.litellm_error_at = None
+        return bool(wrote)
+
+    async def _confirm(self) -> None:
+        """Re-read the LiteLLM deployments straight after a write, so presence is observed, not assumed.
+
+        LiteLLM updates its router inside the /model/new and /model/delete
+        handlers before answering (1.105.0: add_new_model awaits
+        proxy_config.add_deployment; delete_model calls
+        llm_router.delete_deployment), so the read that follows sees the
+        write. If the read fails, the last answer stands and the next skew
+        pass reads again; a failure here never raises out of the probe tick.
+        """
+        try:
+            routes = _routes_from_model_info(await self._model_info_rows())
+        except Exception as e:
+            _LOGGER.warning("the LiteLLM deployments could not be re-read after a write: %s", e)
+            return
+        by_base, pairs = _index_deployments(routes)
+        for dep in self.fleet.deployments:
+            self._mark_presence(dep, routes, by_base, pairs)
 
     async def ensure_providers(self) -> None:
         """At startup: POST /model/new for each config.yaml providers deployment."""
@@ -247,23 +271,53 @@ class DeploymentManager:
         this is the half that says a serving box is never dropped, whatever
         a provider's listing claims.
         """
+        wrote = False
         for dep in self.fleet.deployments:
             if not dep.self_hosted or dep.vllm is None:
                 continue
             srv = dep.vllm
             if srv.state == HEALTHY and dep.api_base and not dep.in_litellm:
-                await self._each(self.add_deployment, dep)
+                wrote = await self._each(self.add_deployment, dep) or wrote
             elif dep.in_litellm and (srv.state in (UNHEALTHY, STALLED) or (srv.unlisted and srv.state != HEALTHY)):
-                await self._each(self.delete_deployment, dep)
+                wrote = await self._each(self.delete_deployment, dep) or wrote
+        if wrote:
+            await self._confirm()
+
+    def _mark_presence(self, dep, routes, by_base, pairs) -> None:
+        """Set what one read of the LiteLLM deployments says about this deployment.
+
+        A provider deployment is present when its (base, model name) pair is.
+        One of our boxes is present when a deployment is at its public port,
+        or at its base under any name (then config_mismatch says which); one
+        at its listen port is noted, not counted.
+        """
+        base = norm_base(dep.api_base) if dep.api_base else None
+        if dep.provider is not None:
+            dep.in_litellm = (base, dep.model_name) in pairs
+            dep.config_mismatch = False
+            return
+        if base is None:
+            # No address is not evidence that LiteLLM lacks it: the last
+            # answer stands. Not even a deployment on the listen port says
+            # otherwise -- off-box that port is unreachable, so it is not the
+            # deployment this server needs.
+            return
+        present = any(
+            dep.vllm is not None and dep.vllm.same_deployment(host, port, model_name)
+            for _norm, host, port, model_name, _id in routes
+        )
+        names = by_base.get(base, [])
+        dep.in_litellm = bool(names) or present
+        dep.config_mismatch = any(name != dep.model_name for name in names)
+        dep.at_listen_port = any(
+            dep.vllm.on_listen_port(host, port, model_name) for _norm, host, port, model_name, _id in routes
+        )
 
     async def check_config_skew(self) -> None:
         resp = await self._transport.request("GET", "model/info")
         self._transport.raise_http(resp)
         routes = _routes_from_model_info(resp.json())
-        by_base: dict[str, list[str]] = {}
-        for norm, _host, _port, model_name, _id in routes:
-            by_base.setdefault(norm, []).append(model_name)
-        pairs = {(norm, model_name) for norm, _host, _port, model_name, _id in routes}
+        by_base, pairs = _index_deployments(routes)
         # One of our boxes serves one model, so its api_base identifies it. A
         # provider's api_base is shared by every model on the account, so a
         # provider deployment is its api_base and model name together.
@@ -280,29 +334,7 @@ class DeploymentManager:
         servers = [d.vllm for d in self.fleet.deployments if d.provider is None and d.vllm is not None]
         now = time.time()
         for dep in self.fleet.deployments:
-            base = norm_base(dep.api_base) if dep.api_base else None
-            if dep.provider is not None:
-                dep.in_litellm = (base, dep.model_name) in pairs
-                dep.config_mismatch = False
-                self._store_skew(dep, now)
-                continue
-            if base is None:
-                # No address is not evidence that LiteLLM lacks it: the last
-                # answer stands. Not even a deployment on the listen port says
-                # otherwise -- off-box that port is unreachable, so it is not the
-                # deployment this server needs.
-                self._store_skew(dep, now)
-                continue
-            present = any(
-                dep.vllm is not None and dep.vllm.same_deployment(host, port, model_name)
-                for _norm, host, port, model_name, _id in routes
-            )
-            names = by_base.get(base, [])
-            dep.in_litellm = bool(names) or present
-            dep.config_mismatch = any(name != dep.model_name for name in names)
-            dep.at_listen_port = any(
-                dep.vllm.on_listen_port(host, port, model_name) for _norm, host, port, model_name, _id in routes
-            )
+            self._mark_presence(dep, routes, by_base, pairs)
             self._store_skew(dep, now)
         extra: list[ExtraLiteLLM] = []
         for norm, host, port, model_name, route_id in routes:
@@ -338,10 +370,11 @@ class DeploymentManager:
                 SKEW_PHRASE[SKEW_IN_LITELLM_NOT_IN_CONFIG],
                 [(row.host, row.port, row.model_name, row.passes) for row in extra],
             )
-        await self._prune_extras(extra)
+        if await self._prune_extras(extra):
+            await self._confirm()
 
-    async def _prune_extras(self, extra: list[ExtraLiteLLM]) -> None:
-        """Delete a route no deployment claims, once it has persisted.
+    async def _prune_extras(self, extra: list[ExtraLiteLLM]) -> int:
+        """Delete a route no deployment claims, once it has persisted. Returns how many were deleted.
 
         This is the deregistration hole in tooling-llmao#29: routes are created
         on the transition into SERVING and deleted on the transition into DOWN,
@@ -370,6 +403,7 @@ class DeploymentManager:
         them after ORPHAN_PASSES skew passes rather than waiting for a
         transition that already happened.
         """
+        pruned = 0
         for row in extra:
             if row.passes < ORPHAN_PASSES:
                 continue
@@ -399,6 +433,8 @@ class DeploymentManager:
                 row.passes,
             )
             self._extra_passes.pop((row.api_base, row.model_name), None)
+            pruned += 1
+        return pruned
 
     async def check_health_skew(self) -> None:
         resp = await self._transport.request("GET", "health")
@@ -450,6 +486,15 @@ class DeploymentManager:
             _LOGGER.warning("skew: %s@%s %s", dep.name, dep.api_base, SKEW_PHRASE[badge])
 
 
+def _index_deployments(rows) -> tuple[dict[str, list[str]], set[tuple[str, str]]]:
+    """The LiteLLM deployments by normalised base (the model names at each), and as (base, model name) pairs."""
+    by_base: dict[str, list[str]] = {}
+    for norm, _host, _port, model_name, _id in rows:
+        by_base.setdefault(norm, []).append(model_name)
+    pairs = {(norm, model_name) for norm, _host, _port, model_name, _id in rows}
+    return by_base, pairs
+
+
 def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str, str]]:
     """Each LiteLLM row as (normalised api_base, host, port, model_name, id).
 
diff --git a/tests/test_deployments.py b/tests/test_deployments.py
index 31b3c8d..98b2d32 100644
--- a/tests/test_deployments.py
+++ b/tests/test_deployments.py
@@ -44,6 +44,31 @@ def _paths(manager):
     return [c.path for c in manager._transport.calls]
 
 
+def _litellm(rows=None):
+    """A stand-in LiteLLM that remembers its deployments, so a read after a write sees the write.
+
+    model/info answers with the rows; model/new appends the posted body with
+    an id; model/delete removes the row with that id. Pass it to _manager.
+    """
+    state = edict(rows=list(rows or []), n=0)
+
+    def answer(call):
+        if call.path == "model/info":
+            return {"data": list(state.rows)}
+        if call.path == "model/new":
+            state.n += 1
+            row = dict(call.json)
+            row["model_info"] = {**(row.get("model_info") or {}), "id": f"id-{state.n}"}
+            state.rows.append(row)
+            return {}
+        if call.path == "model/delete":
+            state.rows = [r for r in state.rows if (r.get("model_info") or {}).get("id") != call.json["id"]]
+            return {}
+        return {}
+
+    return answer
+
+
 def test_deployment_body_from_models():
     s = _server()
     models = {
@@ -85,11 +110,48 @@ def test_sync_selfhost_posts_new_when_serving():
         "gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b")),
     }
     fleet = Fleet(None, [s], models=models)
+    be = _manager(fleet, _litellm())
+    asyncio.run(be.sync_selfhost())
+    # Is it there?, the write, and the read that confirms the write.
+    assert _paths(be) == ["model/info", "model/new", "model/info"]
+    assert fleet.deployments[0].in_litellm is True
+
+
+def test_presence_after_a_write_is_observed_not_assumed():
+    """A LiteLLM that answers 200 to model/new and then does not list the deployment.
+
+    The old code set in_litellm on the strength of the 200. Now the read that
+    follows decides, so the next pass tries again instead of trusting a write
+    LiteLLM lost.
+    """
+    s = _server()
+    s.state = HEALTHY
+    fleet = Fleet(None, [s], models={"gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b"))})
     be = _manager(fleet, {"data": []})
     asyncio.run(be.sync_selfhost())
-    last = be._transport.calls[-1]
-    assert (last.method, last.path) == ("POST", "model/new")
+    assert "model/new" in _paths(be)
+    assert fleet.deployments[0].in_litellm is False
+
+
+def test_a_failed_confirm_keeps_the_last_answer(caplog):
+    """The re-read after a write failed: the write is assumed, as before, and the tick does not raise."""
+    s = _server()
+    s.state = HEALTHY
+    fleet = Fleet(None, [s], models={"gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b"))})
+    reads = []
+
+    def answer(call):
+        if call.path == "model/info":
+            reads.append(call)
+            return (500, {}) if len(reads) > 1 else {"data": []}
+        return {}
+
+    be = _manager(fleet, answer)
+    with caplog.at_level("WARNING", logger="llmao.proxy.deployments"):
+        asyncio.run(be.sync_selfhost())
+    assert _paths(be) == ["model/info", "model/new", "model/info"]
     assert fleet.deployments[0].in_litellm is True
+    assert "could not be re-read after a write" in caplog.text
 
 
 def test_one_failing_deployment_does_not_stop_the_rest(caplog):
@@ -101,7 +163,7 @@ def test_one_failing_deployment_does_not_stop_the_rest(caplog):
     # "gone" is not in models.yaml, so its body cannot be built.
     models = {"gemma4-26b": edict(litellm_params=edict(model="hosted_vllm/gemma4-26b"))}
     fleet = Fleet(None, [broken, *fine], models=models)
-    be = _manager(fleet, {"data": []})
+    be = _manager(fleet, _litellm())
     with caplog.at_level("ERROR", logger="llmao.proxy.deployments"):
         asyncio.run(be.sync_selfhost())
     posted = [c.json["litellm_params"]["api_base"] for c in be._transport.calls if c.path == "model/new"]
@@ -119,8 +181,10 @@ def test_sync_selfhost_deletes_when_down():
     }
     fleet = Fleet(None, [s], models=models)
     fleet.deployments[0].in_litellm = True
-    be = _manager(fleet, _info("10.0.0.1", 8001, model_id="abc"))
+    be = _manager(fleet, _litellm(_info("10.0.0.1", 8001, model_id="abc")["data"]))
     asyncio.run(be.sync_selfhost())
+    # The id lookup, the delete, and the read that confirms it.
+    assert _paths(be) == ["model/info", "model/delete", "model/info"]
     assert fleet.deployments[0].in_litellm is False
 
 
@@ -134,7 +198,7 @@ def test_sync_selfhost_deletes_a_box_no_provider_lists():
     }
     fleet = Fleet(None, [s], models=models)
     fleet.deployments[0].in_litellm = True
-    be = _manager(fleet, _info("10.0.0.1", 8001, model_id="abc"))
+    be = _manager(fleet, _litellm(_info("10.0.0.1", 8001, model_id="abc")["data"]))
     asyncio.run(be.sync_selfhost())
     assert fleet.deployments[0].in_litellm is False
     assert "model/delete" in _paths(be)
@@ -182,6 +246,8 @@ def test_an_orphan_route_is_deleted_once_it_has_persisted():
     assert "model/delete" in _paths(be)
     sent = [c.json["id"] for c in be._transport.calls if c.path == "model/delete"]
     assert sent == ["orphan-1"]
+    # A write, then the read that confirms it.
+    assert _paths(be)[-2:] == ["model/delete", "model/info"]
 
 
 def test_an_orphan_that_is_claimed_again_is_never_deleted():


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-llmao) 02/04: fleet: ORPHAN_PASSES for the skew clock, UNLISTED_PASSES for the listings

Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git

commit b9b5ce3965668cf230cea1e82dabd68c4bf9d475
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 04:49:16 2026 -0500

    fleet: ORPHAN_PASSES for the skew clock, UNLISTED_PASSES for the listings
    
    Early work item 3 of docs/fleet-reconcile-design.md. One constant served
    two clocks: the provider listings, which decide unlisted, and the skew
    passes, which decide when an unclaimed LiteLLM deployment is pruned.
    The orphan prune now has its own count. Both stay 3, and both stay
    constants rather than config keys.
    
    The comment that justified 3 by "half an hour at the 10-minute listing
    interval" is rewritten: the count no longer has to outlast a RunPod
    provisioning gap, because unlisted is an attention verdict, a serving
    box keeps its route, and a box still coming up stays quiet.
    
    Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
 llmao/fleet.py             | 30 +++++++++++++++++++++---------
 llmao/proxy/deployments.py | 11 ++++++-----
 tests/test_deployments.py  |  8 ++++----
 3 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/llmao/fleet.py b/llmao/fleet.py
index dd301db..9e6d5f1 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -129,18 +129,30 @@ RUNPOD_LISTING_INTERVAL_S = 600
 # Boxes already up on other ports. Delete a pair when that box is replaced.
 # Not a template for new hosts. These use the shared selfhost_api_key; every
 # other server uses the key derived from the salt.
-# Consecutive provider listings that must omit a host before llmao concludes
-# its rental ended. Not one: a RunPod pod is absent from the pod list while it
-# provisions -- measured at over 90 minutes after a recreate -- and a single
-# absence therefore means "not yet" as often as it means "gone". Mirrors
-# fleet.health_fail_threshold, which exists for the same reason one failed
-# vLLM probe does not mean a box is down.
+# Two hysteresis counts, one per clock. Each is a constant rather than a config
+# key: no p6 change, no eyaml round-trip. Both mirror fleet.health_fail_threshold,
+# which exists for the same reason: one failed vLLM probe does not mean a box
+# is down, and one absence does not mean a thing is gone.
 #
-# At the 10-minute listing interval this is half an hour of absence, which is
-# longer than any provisioning gap seen and still prompt for a real ending.
-# A constant rather than a config key: no p6 change, no eyaml round-trip.
+# Consecutive provider listings that must omit a host before llmao calls it
+# unlisted (the listing clock). Not one: a RunPod pod is absent from the pod
+# list while it provisions -- measured at over 90 minutes after a recreate --
+# and a single absence means "not yet" as often as "gone". The count does not
+# have to outlast that gap: unlisted is an attention verdict, a serving box
+# keeps its route whatever the listing says, and a box that is still coming
+# up stays quiet (docs/fleet-reconcile-design.md). Today Vast is read on the
+# probe pass and RunPod every RUNPOD_LISTING_INTERVAL_S; the design moves both
+# to one listing interval of about a minute, so three passes is a few minutes.
 UNLISTED_PASSES = 3
 
+# Consecutive skew passes (skew_interval_s, 180s) a LiteLLM deployment that no
+# fleet deployment claims must persist before it is pruned (the skew clock).
+# Absence has innocent causes -- a provider lookup that failed this pass, a box
+# mid-recreate, a deployment llmao is about to claim -- and the counter starts
+# empty on restart, so a proxy full of stale deployments is cleared after this
+# many passes rather than on the first.
+ORPHAN_PASSES = 3
+
 LEGACY_SERVERS = {
     ("80.188.223.202", 10100),  # gemma4-26b, public port from Vast
     ("103.196.86.105", 8004),  # qwen3.8-27b, public port pinned at 15602
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index 4940f73..9c3ea13 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -33,7 +33,7 @@ from llmao.fleet import (
     LEGACY_SERVERS,
     ORDER_SELF_HOSTED,
     ORDER_SPEND,
-    UNLISTED_PASSES,
+    ORPHAN_PASSES,
     ExtraLiteLLM,
     normalize_peer_ip,
     snapshot_for_status,
@@ -365,8 +365,9 @@ class DeploymentManager:
         this pass, a box mid-recreate, a route llmao itself is about to claim.
         The guards above remove the ones that can be named (`unplaced` for a
         server whose address is not known yet), but not the ones that cannot,
-        so UNLISTED_PASSES consecutive passes are required -- the same count,
-        and the same reasoning, as a host missing from a provider listing.
+        so ORPHAN_PASSES consecutive skew passes are required -- its own count
+        on its own clock, for the same reasoning as UNLISTED_PASSES on the
+        provider listings.
 
         No probe gate. An orphan's credential is unknown -- derivation needs the
         LISTEN port and a route only carries the public one -- and the one
@@ -376,11 +377,11 @@ class DeploymentManager:
 
         Deleting is also what makes a restart converge: the counter starts
         empty, so a portal that comes up to a proxy full of stale routes clears
-        them after UNLISTED_PASSES skew passes rather than waiting for a
+        them after ORPHAN_PASSES skew passes rather than waiting for a
         transition that already happened.
         """
         for row in extra:
-            if row.passes < UNLISTED_PASSES:
+            if row.passes < ORPHAN_PASSES:
                 continue
             if not row.route_id:
                 _LOGGER.warning(
diff --git a/tests/test_deployments.py b/tests/test_deployments.py
index 80c93ac..31b3c8d 100644
--- a/tests/test_deployments.py
+++ b/tests/test_deployments.py
@@ -21,7 +21,7 @@ import asyncio
 
 from easydict import EasyDict as edict  # noqa: N813
 
-from llmao.fleet import ORDER_SELF_HOSTED, ORDER_SPEND, UNLISTED_PASSES, Fleet, FleetDeployment
+from llmao.fleet import ORDER_SELF_HOSTED, ORDER_SPEND, ORPHAN_PASSES, Fleet, FleetDeployment
 from llmao.model_status import AWAITING, HEALTHY, SKEW_AT_LISTEN_PORT, UNHEALTHY
 from llmao.providers import ProviderDeployment
 from llmao.proxy.deployments import DeploymentManager
@@ -174,7 +174,7 @@ def test_an_orphan_route_is_deleted_once_it_has_persisted():
     """
     fleet, info = _orphan()
     be = _manager(fleet, info, cfg=edict(fleet=edict(FLEET_KNOBS)))
-    for expected in range(1, UNLISTED_PASSES):
+    for expected in range(1, ORPHAN_PASSES):
         asyncio.run(be.check_config_skew())
         assert [row.passes for row in fleet.extra_litellm] == [expected]
         assert "model/delete" not in _paths(be), "not on the strength of one pass"
@@ -198,7 +198,7 @@ def test_an_orphan_that_is_claimed_again_is_never_deleted():
     assert fleet.extra_litellm == []
     # Back again: the count starts over rather than resuming.
     reply["body"] = info
-    for expected in range(1, UNLISTED_PASSES):
+    for expected in range(1, ORPHAN_PASSES):
         asyncio.run(be.check_config_skew())
         assert [row.passes for row in fleet.extra_litellm] == [expected]
     assert "model/delete" not in _paths(be)
@@ -209,7 +209,7 @@ def test_an_orphan_with_no_litellm_id_is_reported_not_deleted(caplog):
     fleet, info = _orphan(model_id=None)
     be = _manager(fleet, info, cfg=edict(fleet=edict(FLEET_KNOBS)))
     with caplog.at_level("WARNING", logger="llmao.proxy.deployments"):
-        for _ in range(UNLISTED_PASSES + 1):
+        for _ in range(ORPHAN_PASSES + 1):
             asyncio.run(be.check_config_skew())
     assert "model/delete" not in _paths(be)
     assert "has no LiteLLM id" in caplog.text


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-llmao) 01/04: reconcile: the deployment key, frozen with tests

Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git

commit 9fbee0efa4aa170eefe2c1e4aa79cfbeaad947f2
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 04:44:43 2026 -0500

    reconcile: the deployment key, frozen with tests
    
    Migration step 1 of docs/fleet-reconcile-design.md. One function builds
    every deployment's identity: a canonical base URL and our model name.
    A box is keyed by its listen port, so a provider's port remap keeps the
    identity; a provider deployment by its config.yaml base, so two of our
    names on one provider model are two keys.
    
    llmao/reconcile.py starts as the pure bottom layer the design names.
    It takes norm_base from the deployment manager (now also lowercasing
    scheme and host, and unwrapping an IPv4-mapped IPv6 host) and
    normalize_peer_ip from fleet.py, which re-exports it. VllmServer and
    FleetDeployment gain a `key` property; nothing reads it yet.
    
    DeploymentKey is a NamedTuple rather than an edict: the key joins
    records across sources, so it must hash, and a dict does not.
    
    Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
 llmao/fleet.py             |  20 ++++---
 llmao/proxy/deployments.py |  46 +++++----------
 llmao/reconcile.py         | 123 ++++++++++++++++++++++++++++++++++++++++
 tests/test_fleet.py        |  39 -------------
 tests/test_reconcile.py    | 136 +++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 285 insertions(+), 79 deletions(-)

diff --git a/llmao/fleet.py b/llmao/fleet.py
index 2dbd8a1..dd301db 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -52,6 +52,7 @@ from llmao.model_status import (
 )
 from llmao.models import load_models, served_model_name, validate_models, vllm_recipe, warn_deprecated
 from llmao.providers import ProviderDeployment, gpu_host_api_key, provider_deployments, validate_providers
+from llmao.reconcile import DeploymentKey, deployment_key, normalize_peer_ip
 from llmao.runpod_client import boxes_from_pods, fetch_pods
 from llmao.vast_client import boxes_from_body, fetch_instances, port_map_from_body
 
@@ -82,13 +83,6 @@ def _ago(ts: float | None) -> str:
     return f"{sec // 3600}h"
 
 
-def normalize_peer_ip(addr: str | None) -> str:
-    ip = (addr or "").strip()
-    if ip.startswith("::ffff:"):
-        ip = ip[7:]
-    return ip
-
-
 # Box-claimed public IP (Vast PUBLIC_IPADDR). Not X-Forwarded-For.
 HOST_HEADER = "X-LLMAO-Host"
 
@@ -463,6 +457,11 @@ class VllmServer:
         """Human-readable 'config_served_at' timestamp."""
         return _ago(self.config_served_at)
 
+    @property
+    def key(self) -> DeploymentKey:
+        """This server's identity everywhere: its listen URL and model name. Never the public port."""
+        return deployment_key(self.model_name, host=self.host, listen_port=self.listen_port)
+
     @property
     def root_url(self) -> str | None:
         if self.public_port is None:
@@ -672,6 +671,13 @@ class FleetDeployment:
             litellm_model=pd.litellm_model,
         )
 
+    @property
+    def key(self) -> DeploymentKey:
+        """The deployment's identity: the server's listen URL, or the provider's base, plus our model name."""
+        if self.vllm is not None:
+            return self.vllm.key
+        return deployment_key(self.model_name, api_base=self._api_base)
+
     @property
     def api_base(self) -> str | None:
         if self.vllm is not None:
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index acb8767..4940f73 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -49,6 +49,7 @@ from llmao.model_status import (
 )
 from llmao.providers import provider_api_key
 from llmao.proxy.errors import BackendUnavailableError
+from llmao.reconcile import norm_base
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -100,7 +101,7 @@ class DeploymentManager:
 
         info: dict[str, Any] = {
             "self_hosted": dep.self_hosted,
-            "asf_api_base": _norm_base(dep.api_base),
+            "asf_api_base": norm_base(dep.api_base),
         }
         if dep.access_groups:
             # What LiteLLM reads to decide which teams may use this
@@ -134,13 +135,13 @@ class DeploymentManager:
     def _litellm_id(self, row: dict, dep) -> str | None:
         if str(row.get("model_name") or "") != dep.model_name:
             return None
-        want = _norm_base(dep.api_base)
+        want = norm_base(dep.api_base)
         info = row.get("model_info") or {}
         if not isinstance(info, dict):
             info = {}
         if (
-            _norm_base(info.get("asf_api_base")) == want
-            or _norm_base(
+            norm_base(info.get("asf_api_base")) == want
+            or norm_base(
                 (row.get("litellm_params") or {}).get("api_base") if isinstance(row.get("litellm_params"), dict) else ""
             )
             == want
@@ -169,7 +170,7 @@ class DeploymentManager:
         if dep.provider is not None:
             # Registered at every startup, so a restart must find the last one
             # rather than add a second copy.
-            want = (_norm_base(dep.api_base), dep.model_name)
+            want = (norm_base(dep.api_base), dep.model_name)
             return any((norm, model_name) == want for norm, _host, _port, model_name, _id in routes)
         if not dep.self_hosted or dep.vllm is None:
             return False
@@ -276,8 +277,8 @@ class DeploymentManager:
         # One of our boxes serves one model, so its api_base identifies it. A
         # provider's api_base is shared by every model on the account, so a
         # provider deployment is its api_base and model name together.
-        ours = {_norm_base(d.api_base) for d in self.fleet.deployments if d.api_base and d.provider is None}
-        bought = {(_norm_base(d.api_base), d.model_name) for d in self.fleet.deployments if d.provider is not None}
+        ours = {norm_base(d.api_base) for d in self.fleet.deployments if d.api_base and d.provider is None}
+        bought = {(norm_base(d.api_base), d.model_name) for d in self.fleet.deployments if d.provider is not None}
         # Servers whose address is not known this pass: no public port yet, or
         # the provider's lookup failed. A LiteLLM deployment on the same host
         # and model may well be theirs, so it is not called an extra.
@@ -289,7 +290,7 @@ class DeploymentManager:
         servers = [d.vllm for d in self.fleet.deployments if d.provider is None and d.vllm is not None]
         now = time.time()
         for dep in self.fleet.deployments:
-            base = _norm_base(dep.api_base) if dep.api_base else None
+            base = norm_base(dep.api_base) if dep.api_base else None
             if dep.provider is not None:
                 dep.in_litellm = (base, dep.model_name) in pairs
                 dep.config_mismatch = False
@@ -415,12 +416,12 @@ class DeploymentManager:
         # Each endpoint as (api_base, LiteLLM model). Our boxes match on the
         # base alone; a provider's base is shared, so it matches on both.
         healthy = {
-            (_norm_base(x.get("api_base")), x.get("model"))
+            (norm_base(x.get("api_base")), x.get("model"))
             for x in (body.get("healthy_endpoints") or [])
             if isinstance(x, dict) and x.get("api_base")
         }
         unhealthy = {
-            (_norm_base(x.get("api_base")), x.get("model"))
+            (norm_base(x.get("api_base")), x.get("model"))
             for x in (body.get("unhealthy_endpoints") or [])
             if isinstance(x, dict) and x.get("api_base")
         }
@@ -428,7 +429,7 @@ class DeploymentManager:
         unhealthy_bases = {base for base, _model in unhealthy}
         stamp = time.time()
         for dep in self.fleet.deployments:
-            base = _norm_base(dep.api_base) if dep.api_base else None
+            base = norm_base(dep.api_base) if dep.api_base else None
             if base is None:
                 # Nothing to match LiteLLM's answer to. The last result stands
                 # rather than being wiped for the four hours until the next check.
@@ -458,27 +459,6 @@ class DeploymentManager:
             _LOGGER.warning("skew: %s@%s %s", dep.name, dep.api_base, SKEW_PHRASE[badge])
 
 
-def _norm_base(url: Any) -> str:
-    """Strip trailing slashes and a trailing /v1.
-
-    The identity the skew check compares on: the same route appears with the
-    suffix (VllmServer.api_base, and the litellm_params.api_base copied from
-    it) and without (asf_api_base, routes pushed before the suffix). The
-    comparison must match across both forms.
-
-    (The suffix was once actively wrong: an older LiteLLM appended
-    /v1/chat/completions to api_base, so a /v1-suffixed base resolved to
-    /v1/v1/..., 404'd at the origin, and surfaced as an OpenAI
-    authentication error naming platform.openai.com -- which sends the reader
-    after their own key rather than the route. The current LiteLLM appends
-    only /chat/completions, so the suffix is what vLLM needs now.)
-    """
-    s = str(url or "").rstrip("/")
-    if s.endswith("/v1"):
-        s = s[:-3].rstrip("/")
-    return s
-
-
 def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str, str]]:
     """Each LiteLLM row as (normalised api_base, host, port, model_name, id).
 
@@ -499,7 +479,7 @@ def _routes_from_model_info(body: Any) -> list[tuple[str, str, int | None, str,
         params = row.get("litellm_params") or {}
         if not isinstance(params, dict) or not params.get("api_base"):
             continue
-        norm = _norm_base(params["api_base"])
+        norm = norm_base(params["api_base"])
         parsed = urlparse(norm)
         info = row.get("model_info") or {}
         route_id = str(info.get("id") or "") if isinstance(info, dict) else ""
diff --git a/llmao/reconcile.py b/llmao/reconcile.py
new file mode 100644
index 0000000..1a12e24
--- /dev/null
+++ b/llmao/reconcile.py
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Fleet reconciliation: the pure part. See docs/fleet-reconcile-design.md.
+
+Today this holds the deployment key. The snapshots, the record and
+``reconcile()`` arrive here as the design is worked through, and
+``model_status.py`` is absorbed then.
+
+Pure: nothing here does I/O, and nothing here imports ``llmao.fleet``.
+"""
+
+from __future__ import annotations
+
+from typing import NamedTuple
+from urllib.parse import urlparse
+
+
+def normalize_peer_ip(addr: str | None) -> str:
+    """A host as the box protocol keys it: trimmed, IPv4-mapped IPv6 unwrapped."""
+    ip = (addr or "").strip()
+    if ip.startswith("::ffff:"):
+        ip = ip[7:]
+    return ip
+
+
+def norm_base(url: object) -> str:
+    """The canonical form of a base URL.
+
+    The same deployment turns up with a trailing ``/v1`` (``VllmServer.api_base``,
+    and the ``litellm_params.api_base`` copied from it) and without it
+    (``asf_api_base``, routes pushed before the suffix), with and without a
+    trailing slash, and with the host in whatever case the hand that typed it
+    used. Comparisons must match across all of those, so: scheme and host
+    lowercased, an IPv4-mapped IPv6 host unwrapped, trailing slashes gone, and
+    one trailing ``/v1`` gone. Only a trailing ``/v1``: a path that merely
+    contains it is left alone.
+
+    (The suffix was once actively wrong: an older LiteLLM appended
+    ``/v1/chat/completions`` to api_base, so a ``/v1``-suffixed base resolved
+    to ``/v1/v1/...``, 404'd at the origin, and surfaced as an OpenAI
+    authentication error naming platform.openai.com, which sends the reader
+    after their own key rather than the route. The current LiteLLM appends only
+    ``/chat/completions``, so the suffix is what vLLM needs now.)
+    """
+    s = str(url or "").strip().rstrip("/")
+    if s.endswith("/v1"):
+        s = s[:-3].rstrip("/")
+    parsed = urlparse(s)
+    if not parsed.scheme or not parsed.netloc:
+        # Not a URL. The string rules above are all that can be applied.
+        return s
+    host = normalize_peer_ip(parsed.hostname)
+    if ":" in host:
+        host = f"[{host}]"
+    netloc = f"{host}:{parsed.port}" if parsed.port else host
+    return f"{parsed.scheme.lower()}://{netloc}{parsed.path}"
+
+
+class DeploymentKey(NamedTuple):
+    """What every source calls one deployment: a canonical base URL and our model name.
+
+    One shape for our boxes and for the providers we buy from. A box's URL is
+    built from its LISTEN port, not the public port a provider maps it to, so a
+    port remap keeps the identity. The model name is our public ``model_name``,
+    not the provider's model string, so two of our names on one provider model
+    are two keys.
+
+    A NamedTuple, not the edict the rest of the codebase prefers: the key's job
+    is to join records across sources, so it is used as a dict key and a set
+    member, and an edict is a dict, which is unhashable. A NamedTuple hashes,
+    compares as a tuple, cannot be changed after it is built, and still reads
+    as ``key.url`` and ``key.model_name``.
+    """
+
+    url: str
+    model_name: str
+
+
+def deployment_key(
+    model_name: str,
+    *,
+    host: str | None = None,
+    listen_port: int | None = None,
+    api_base: str | None = None,
+) -> DeploymentKey:
+    """The one function that builds a key.
+
+    A box: ``host`` and ``listen_port``. A provider: ``api_base``, the base
+    from config.yaml. Either way the URL goes through ``norm_base``, so the
+    stored key, the comparison, and the stamp in LiteLLM's ``model_info`` are
+    the same bytes.
+    """
+    name = str(model_name or "").strip()
+    if not name:
+        raise ValueError("deployment_key: model_name is required")
+    if api_base is not None:
+        if host is not None or listen_port is not None:
+            raise ValueError("deployment_key: api_base or host and listen_port, not both")
+        url = norm_base(api_base)
+        if not url:
+            raise ValueError("deployment_key: api_base is empty")
+        return DeploymentKey(url, name)
+    peer = normalize_peer_ip(host)
+    if not peer or listen_port is None:
+        raise ValueError("deployment_key: host and listen_port are required for a box")
+    if ":" in peer:
+        peer = f"[{peer}]"
+    return DeploymentKey(norm_base(f"http://{peer}:{int(listen_port)}"), name)
diff --git a/tests/test_fleet.py b/tests/test_fleet.py
index aa164d0..e274325 100644
--- a/tests/test_fleet.py
+++ b/tests/test_fleet.py
@@ -38,7 +38,6 @@ from llmao.fleet import (
 )
 from llmao.model_status import AWAITING, HEALTHY, LOADING, deployment_status
 from llmao.models import load_models, vllm_recipe
-from llmao.proxy.deployments import _norm_base
 from llmao.vllm_api_key import derive_vllm_api_key
 
 EXAMPLE = Path(__file__).resolve().parent.parent / "models.yaml"
@@ -486,44 +485,6 @@ def test_client_ip_claimed_host_not_peer():
     assert HOST_HEADER == "X-LLMAO-Host"
 
 
-def test_norm_base_strips_trailing_v1():
-    """Why the normalisation exists.
-
-    An older LiteLLM appended /v1/chat/completions to api_base, so a base
-    that already ended in /v1 resolved to /v1/v1/... and 404'd at the origin.
-    LiteLLM then reported an OpenAI authentication error naming
-    platform.openai.com -- so the symptom pointed at the caller's key rather
-    than at the route. This happened in production against the Fastly-fronted
-    Gemma endpoint. The current LiteLLM appends only /chat/completions, so a
-    /v1-suffixed base is what vLLM needs now; routes pushed under the old
-    behaviour may lack it, and the comparison must match across both forms.
-    """
-    assert _norm_base("https://llm.tooling.apache.org/v1") == "https://llm.tooling.apache.org"
-    assert _norm_base("https://llm.tooling.apache.org/v1/") == "https://llm.tooling.apache.org"
-    assert _norm_base("http://127.0.0.1:8003/v1") == "http://127.0.0.1:8003"
-
-
-def test_norm_base_leaves_other_paths_alone():
-    """Only a trailing /v1 is stripped -- not a path that merely contains it."""
-    assert _norm_base("http://127.0.0.1:8003") == "http://127.0.0.1:8003"
-    assert _norm_base("http://127.0.0.1:8003/") == "http://127.0.0.1:8003"
-    assert _norm_base("https://example.com/v1/proxy") == "https://example.com/v1/proxy"
-    assert _norm_base("") == ""
-    assert _norm_base(None) == ""
-
-
-def test_norm_base_makes_skew_comparison_match():
-    """The comparison this exists for.
-
-    check_config_skew compares VllmServer.api_base against what LiteLLM reports.
-    VllmServer.api_base ends in /v1; a deployment pushed before that, or a
-    models.yaml row, may lack it. Without normalisation on both sides the
-    same deployment never matches, and the skew check reports a missing
-    deployment that is actually present.
-    """
-    assert _norm_base("http://100.105.28.100:8003/v1") == _norm_base("http://100.105.28.100:8003")
-
-
 def test_from_row_api_key_is_salt_hmac():
     """Box JSON carries the salt HMAC, not the legacy shared bearer."""
     cfg = _cfg({"10.0.0.1": [["qwen3-8b", 8003]]})
diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py
new file mode 100644
index 0000000..df1be38
--- /dev/null
+++ b/tests/test_reconcile.py
@@ -0,0 +1,136 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""The deployment key, frozen: docs/fleet-reconcile-design.md, migration step 1."""
+
+import pytest
+
+from llmao.reconcile import DeploymentKey, deployment_key, norm_base
+
+
+def test_norm_base_strips_trailing_v1():
+    """Why the normalisation exists.
+
+    An older LiteLLM appended /v1/chat/completions to api_base, so a base
+    that already ended in /v1 resolved to /v1/v1/... and 404'd at the origin.
+    LiteLLM then reported an OpenAI authentication error naming
+    platform.openai.com -- so the symptom pointed at the caller's key rather
+    than at the route. This happened in production against the Fastly-fronted
+    Gemma endpoint. The current LiteLLM appends only /chat/completions, so a
+    /v1-suffixed base is what vLLM needs now; routes pushed under the old
+    behaviour may lack it, and the comparison must match across both forms.
+    """
+    assert norm_base("https://llm.tooling.apache.org/v1") == "https://llm.tooling.apache.org"
+    assert norm_base("https://llm.tooling.apache.org/v1/") == "https://llm.tooling.apache.org"
+    assert norm_base("http://127.0.0.1:8003/v1") == "http://127.0.0.1:8003"
+
+
+def test_norm_base_leaves_other_paths_alone():
+    """Only a trailing /v1 is stripped -- not a path that merely contains it, and only one."""
+    assert norm_base("http://127.0.0.1:8003") == "http://127.0.0.1:8003"
+    assert norm_base("https://openrouter.ai/api/v1") == "https://openrouter.ai/api"
+    assert norm_base("https://example.test/v1/proxy") == "https://example.test/v1/proxy"
+    assert norm_base("https://example.test/v1/v1") == "https://example.test/v1"
+
+
+def test_norm_base_host_case_and_mapped_ipv6():
+    """Scheme and host are case-insensitive on the wire, so they are one key here.
+
+    A peer that arrived as IPv4-mapped IPv6 is the same box as its IPv4 form.
+    """
+    assert norm_base("HTTPS://OpenRouter.ai/api/v1") == "https://openrouter.ai/api"
+    assert norm_base("http://[::ffff:192.0.2.10]:8001/v1") == "http://192.0.2.10:8001"
+    assert norm_base("http://[2001:db8::1]:8001") == "http://[2001:db8::1]:8001"
+
+
+def test_norm_base_makes_skew_comparison_match():
+    """The comparison this exists for.
+
+    check_config_skew compares VllmServer.api_base against what LiteLLM reports.
+    VllmServer.api_base ends in /v1; a deployment pushed before that, or a
+    models.yaml row, may lack it. Without normalisation on both sides the
+    same deployment never matches, and the skew check reports a missing
+    deployment that is actually present.
+    """
+    assert norm_base("http://192.0.2.10:8003/v1") == norm_base("http://192.0.2.10:8003")
+
+
+def test_norm_base_tolerates_what_is_not_a_url():
+    assert norm_base(None) == ""
+    assert norm_base("") == ""
+    assert norm_base("  ") == ""
+    assert norm_base("192.0.2.10:8001/v1") == "192.0.2.10:8001"
+
+
+def test_box_key_is_the_listen_port():
+    """A Vast remap changes the public port and nothing else; the key does not move."""
+    key = deployment_key("gemma4-26b", host="192.0.2.10", listen_port=8001)
+    assert key == DeploymentKey("http://192.0.2.10:8001", "gemma4-26b")
+    assert key.url == "http://192.0.2.10:8001"
+    assert key.model_name == "gemma4-26b"
+    assert deployment_key("gemma4-26b", host="::ffff:192.0.2.10", listen_port="8001") == key
+
+
+def test_box_key_matches_a_stamped_or_registered_base():
+    """What LiteLLM holds, in any of its forms, normalises to the same bytes."""
+    key = deployment_key("qwen3-8b", host="192.0.2.10", listen_port=8003)
+    for form in ("http://192.0.2.10:8003/v1", "http://192.0.2.10:8003/", "HTTP://192.0.2.10:8003"):
+        assert norm_base(form) == key.url
+
+
+def test_legacy_pair_is_an_ordinary_key():
+    """A legacy box on an odd listen port is keyed like any other: nothing remembers it is legacy."""
+    key = deployment_key("gemma4-26b", host="192.0.2.20", listen_port=10100)
+    assert key.url == "http://192.0.2.20:10100"
+
+
+def test_sibling_ports_are_two_keys():
+    a = deployment_key("gemma4-26b", host="192.0.2.10", listen_port=8001)
+    b = deployment_key("qwen3-8b", host="192.0.2.10", listen_port=8002)
+    assert a != b
+    assert {a, b} == {a, b, a}
+
+
+def test_provider_key_is_the_base_and_our_name():
+    key = deployment_key("grok-4", api_base="https://openrouter.ai/api/v1")
+    assert key == DeploymentKey("https://openrouter.ai/api", "grok-4")
+
+
+def test_two_names_on_one_provider_model_are_two_keys():
+    """grok-4 and grok-4-now may be one upstream model; they are two deployments to us."""
+    base = "https://api.x.ai/v1"
+    assert deployment_key("grok-4", api_base=base) != deployment_key("grok-4-now", api_base=base)
+
+
+def test_same_name_from_two_providers_are_two_keys():
+    """grok-4 from xAI and from OpenRouter: one name, two bases, two keys."""
+    assert deployment_key("grok-4", api_base="https://api.x.ai/v1") != deployment_key(
+        "grok-4", api_base="https://openrouter.ai/api/v1"
+    )
+
+
+def test_key_refuses_half_an_identity():
+    with pytest.raises(ValueError):
+        deployment_key("", host="192.0.2.10", listen_port=8001)
+    with pytest.raises(ValueError):
+        deployment_key("gemma4-26b", host="192.0.2.10")
+    with pytest.raises(ValueError):
+        deployment_key("gemma4-26b", listen_port=8001)
+    with pytest.raises(ValueError):
+        deployment_key("grok-4", api_base="")
+    with pytest.raises(ValueError):
+        deployment_key("grok-4", api_base="https://api.x.ai/v1", host="192.0.2.10")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-llmao) 03/04: fleet: one working-key rule, shared by the deployment body and /v1/models

Posted by gs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git

commit 039c403b78857681c5022071239bfed191cd47b9
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Wed Oct 21 06:01:17 2026 -0500

    fleet: one working-key rule, shared by the deployment body and /v1/models
    
    Early work item 5 of docs/fleet-reconcile-design.md. The rule that picks
    the bearer a box accepts (shared fleet.selfhost_api_key for the legacy
    set, the salt-derived key for every other) lived on the deployment
    manager, so only the LiteLLM registration used it. The served-window
    read on the edge into Healthy sent the derived key to every box, and a
    legacy box answered 401 on every pass and never reported its context
    window. Under the design's Foreign rule that same 401 would have
    deleted the deployment.
    
    working_api_key(cfg, srv) now lives in fleet.py; the executor's method
    calls it and the fetch_max_model_len call sends it. A test drives
    probe_all against a legacy host and checks the bearer that goes out.
    
    Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
---
 llmao/fleet.py             | 18 +++++++++++++-
 llmao/proxy/deployments.py | 20 ++++------------
 tests/test_fleet.py        | 58 +++++++++++++++++++++++++++++++++++++++-------
 3 files changed, 71 insertions(+), 25 deletions(-)

diff --git a/llmao/fleet.py b/llmao/fleet.py
index 9e6d5f1..1612f63 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -165,6 +165,22 @@ def _listen_port_allowed(host: str, port: int) -> bool:
     return port in VLLM_LISTEN_PORTS or (normalize_peer_ip(host), port) in LEGACY_SERVERS
 
 
+def working_api_key(cfg: Any, srv: Any) -> str:
+    """The bearer this vLLM process accepts: what LiteLLM registers and what /v1/models is asked with.
+
+    LEGACY_SERVERS were launched with the shared fleet.selfhost_api_key, which
+    validate_fleet already required when one of those hosts is configured.
+    Every other box uses the key derived from the salt (srv.api_key). One
+    rule, read by the deployment body and by the served-window fetch alike;
+    the fetch used to send the derived key to every box, so a legacy box
+    answered 401 on every pass and never reported its context window.
+    No probe: the wrong key 401s, and the next pass says so.
+    """
+    if (normalize_peer_ip(srv.host), int(srv.listen_port)) in LEGACY_SERVERS:
+        return str(cfg.fleet.selfhost_api_key).strip()
+    return str(srv.api_key or "")
+
+
 def _require_listen_port(host: str, port: int) -> None:
     if _listen_port_allowed(host, port):
         return
@@ -1092,7 +1108,7 @@ class Fleet:
                 if srv.state == HEALTHY and (was != HEALTHY or srv.observed_max_model_len is None):
                     root = srv.root_url
                     if root:
-                        mml = await fetch_max_model_len(client, root, srv.api_key)
+                        mml = await fetch_max_model_len(client, root, working_api_key(self.cfg, srv))
                         if mml is not None:
                             srv.observed_max_model_len = mml
                             srv.observed_at = stamp
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index 9c3ea13..e84850c 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -30,13 +30,13 @@ from typing import Any
 from urllib.parse import urlparse
 
 from llmao.fleet import (
-    LEGACY_SERVERS,
     ORDER_SELF_HOSTED,
     ORDER_SPEND,
     ORPHAN_PASSES,
     ExtraLiteLLM,
     normalize_peer_ip,
     snapshot_for_status,
+    working_api_key,
 )
 from llmao.model_status import (
     HEALTHY,
@@ -149,21 +149,11 @@ class DeploymentManager:
             return str(info.get("id") or row.get("model_id") or "") or None
         return None
 
-    async def _working_api_key(self, dep) -> str | None:
-        """The bearer to register for this server.
-
-        LEGACY_SERVERS were launched with the shared selfhost_api_key.
-        validate_fleet already required that key when one of those hosts is
-        configured, so this just returns it. Every other box uses the key
-        derived from the salt. No probe: trying the wrong one 401s and the
-        next pass deletes the route.
-        """
+    def _working_api_key(self, dep) -> str | None:
+        """The bearer to register for this server: fleet.working_api_key, or None off our boxes."""
         if not dep.self_hosted or dep.vllm is None:
             return None
-        srv = dep.vllm
-        if (normalize_peer_ip(srv.host), int(srv.listen_port)) in LEGACY_SERVERS:
-            return str(self._cfg.fleet.selfhost_api_key).strip()
-        return srv.api_key or None
+        return working_api_key(self._cfg, dep.vllm) or None
 
     async def _deployment_present(self, dep) -> bool:
         routes = _routes_from_model_info(await self._model_info_rows())
@@ -188,7 +178,7 @@ class DeploymentManager:
             dep.in_litellm = True
             _LOGGER.info("model/new already present %s@%s", dep.name, dep.api_base)
             return
-        body = self.deployment_body(dep, api_key=await self._working_api_key(dep))
+        body = self.deployment_body(dep, api_key=self._working_api_key(dep))
         resp = await self._transport.request("POST", "model/new", json=body)
         if resp.status_code in (400, 409):
             dep.in_litellm = True
diff --git a/tests/test_fleet.py b/tests/test_fleet.py
index e274325..e293975 100644
--- a/tests/test_fleet.py
+++ b/tests/test_fleet.py
@@ -35,6 +35,7 @@ from llmao.fleet import (
     parse_host_row,
     snapshot_for_status,
     validate_fleet,
+    working_api_key,
 )
 from llmao.model_status import AWAITING, HEALTHY, LOADING, deployment_status
 from llmao.models import load_models, vllm_recipe
@@ -388,6 +389,7 @@ def test_legacy_boxes_need_the_shared_key_and_keep_their_ports():
 
 
 def test_working_api_key_follows_the_legacy_set():
+    """One rule for the bearer a box accepts: shared for LEGACY_SERVERS, derived for every other."""
     from llmao.proxy.client import LiteLLMBackend
 
     cfg = _cfg({"80.188.223.202": [["gemma4-26b", 10100]]})
@@ -396,18 +398,56 @@ def test_working_api_key_follows_the_legacy_set():
     models = load_models(EXAMPLE)
     validate_fleet(cfg, models=models)
     fleet = Fleet.from_cfg(cfg, models=models)
-    backend = LiteLLMBackend(cfg, fleet)
     legacy = fleet.deployments[0]
-    other = edict(self_hosted=True, vllm=edict(host="10.0.0.1", listen_port=8001, api_key="sk-derived"))
-    in_range = edict(self_hosted=True, vllm=edict(host="213.173.110.78", listen_port=8003, api_key="sk-derived"))
+    other = edict(host="10.0.0.1", listen_port=8001, api_key="sk-derived")
+    in_range = edict(host="213.173.110.78", listen_port=8003, api_key="sk-derived")
+    assert working_api_key(cfg, legacy.vllm) == "sk-shared"
+    assert working_api_key(cfg, in_range) == "sk-shared"
+    assert working_api_key(cfg, other) == "sk-derived"
 
-    async def run():
-        assert await backend.deployments._working_api_key(legacy) == "sk-shared"
-        assert await backend.deployments._working_api_key(in_range) == "sk-shared"
-        assert await backend.deployments._working_api_key(other) == "sk-derived"
-        await backend.aclose()
+    # The executor asks the same function, and has no answer off our boxes.
+    backend = LiteLLMBackend(cfg, fleet)
+    assert backend.deployments._working_api_key(legacy) == "sk-shared"
+    assert backend.deployments._working_api_key(edict(self_hosted=False, vllm=None)) is None
+    asyncio.run(backend.aclose())
 
-    asyncio.run(run())
+
+def test_served_window_is_asked_for_with_the_working_key():
+    """The /v1/models read on the edge into Healthy sends what the box accepts.
+
+    It used to send the derived key to every box, so a legacy box answered
+    401 on every pass and its context window stayed unknown for good.
+    """
+    cfg = _cfg({"80.188.223.202": [["gemma4-26b", 10100]]})
+    cfg.fleet.selfhost_api_key = "sk-shared"
+    models = load_models(EXAMPLE)
+    validate_fleet(cfg, models=models)
+    fleet = Fleet.from_cfg(cfg, models=models)
+    fleet.apply_port_map({"80.188.223.202": {"10100": 40000}})
+    (srv,) = fleet.servers
+    bearers = []
+
+    class _Resp:
+        def __init__(self, status_code, body=None):
+            self.status_code = status_code
+            self._body = body
+            self.text = "OK"
+
+        def json(self):
+            return self._body
+
+    class _Client:
+        async def get(self, url, headers=None):
+            if url.endswith("/health"):
+                return _Resp(200)
+            bearers.append((headers or {}).get("Authorization"))
+            if headers and headers.get("Authorization") == "Bearer sk-shared":
+                return _Resp(200, {"data": [{"max_model_len": 8192}]})
+            return _Resp(401)
+
+    asyncio.run(fleet.probe_all(client=_Client(), now=1.0))
+    assert bearers == ["Bearer sk-shared"]
+    assert srv.observed_max_model_len == 8192
 
 
 def test_listen_port_outside_the_set_fails():


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org