You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/16 18:11:13 UTC
(tooling-llmao) 03/05: fleet: compare with LiteLLM only after the fleet's first pass
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit bc1d40abfa362f41d701a75036d1cefffcbc47cf
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Fri Oct 16 12:30:05 2026 -0500
fleet: compare with LiteLLM only after the fleet's first pass
After a restart, the "Work needed" strip on /models listed every
Vast-mapped deployment as "in LiteLLM, not in config", and one box as
"intended, not in LiteLLM", while the Deployments section of the same
page showed all of them Healthy.
llmao starts two loops together. fleet-lifecycle looks up the public
ports from Vast's API, probes every server, and syncs registrations.
litellm-skew compares the fleet with LiteLLM at once, then every three
minutes, and on that same first pass runs LiteLLM's health check, which
does not run again for four hours. It ran before any Vast server had a
public port: with no api_base, no LiteLLM deployment matched its
server. The config badges cleared on the next pass; the health check's
blind answer stood for four hours. Servers whose public port is pinned
in fleet.hosts were known from the start, so they were never listed.
Fleet.ready is set at the end of the lifecycle's first pass, and
run_skew waits for it once before its first comparison. It is set even
after a failed pass: a comparison waiting on a fleet that never answers
would stop for good.
A deployment whose address goes missing later in the run is not yet
handled; that is the next change.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
llmao/fleet.py | 8 ++++++++
llmao/proxy/client.py | 5 +++++
tests/test_fleet_health.py | 45 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 58 insertions(+)
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 253aab1..fa73a5c 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -684,6 +684,11 @@ class Fleet:
# Awaitable after each vLLM probe tick (DeploymentManager.sync_selfhost).
# Wired from create_app so this module does not import the LiteLLM client.
self.after_probe = after_probe
+ # Set when the first lifecycle pass is over: public ports looked up,
+ # every server probed, registrations synced. Whatever compares the
+ # fleet with LiteLLM waits for it; before then no Vast server has an
+ # address, and a comparison reads every one of them as missing.
+ self.ready = asyncio.Event()
self.config_fetch_at: dict[str, float] = {}
# Fleet-key GETs for IPs not in fleet.hosts. Process memory only.
self.unknown_config_fetches: dict[str, edict] = {}
@@ -1127,6 +1132,9 @@ class Fleet:
await self.after_probe()
except Exception:
_LOGGER.exception("fleet lifecycle probe failed")
+ # Even after a failed pass: waiting on a fleet that never answers
+ # would stop the comparisons for good, and they report the failure.
+ self.ready.set()
await asyncio.sleep(interval)
diff --git a/llmao/proxy/client.py b/llmao/proxy/client.py
index 0044104..3e651e0 100644
--- a/llmao/proxy/client.py
+++ b/llmao/proxy/client.py
@@ -395,6 +395,11 @@ class LiteLLMBackend:
health_s = float(self._cfg.fleet.litellm_health_interval_s)
last_health = 0.0
first = True
+ # Not before the fleet's first pass. Started together, this ran first:
+ # no Vast server had its public port yet, so every one of their LiteLLM
+ # deployments read as "not in config", and the 4-hour health check
+ # that runs on the first pass had nothing to match its answers to.
+ await self.deployments.fleet.ready.wait()
while True:
try:
await self.deployments.check_config_skew()
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index 4956fc2..69838d8 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -590,3 +590,48 @@ def test_probe_all_scrapes_from_root_not_api_base():
assert s.kv_cache_tokens == 855836
assert s.observed_max_model_len == 131072
assert calls == [f"{root}/health", f"{root}/metrics", f"{root}/v1/models"]
+
+
+def test_the_fleet_is_ready_after_its_first_pass_even_a_failed_one():
+ """A comparison that waited on a fleet whose first pass failed would never run."""
+ fleet = Fleet(edict(fleet=edict(FLEET_KNOBS)), [])
+
+ async def fail(*_args, **_kwargs):
+ raise RuntimeError("Vast API unreachable")
+
+ fleet.refresh_public_ports = fail
+ fleet.probe_all = fail
+
+ async def scenario():
+ task = asyncio.create_task(fleet.run_lifecycle())
+ try:
+ await asyncio.wait_for(fleet.ready.wait(), timeout=5)
+ finally:
+ task.cancel()
+
+ assert fleet.ready.is_set() is False
+ asyncio.run(scenario())
+ assert fleet.ready.is_set()
+
+
+def test_litellm_is_not_compared_with_the_fleet_before_its_first_pass():
+ from llmao.proxy.client import LiteLLMBackend
+
+ fleet = Fleet(edict(fleet=edict(FLEET_KNOBS)), [])
+ fake = FakeTransport(lambda call: {"data": []} if call.path == "model/info" else {})
+ backend = LiteLLMBackend(edict(fleet=edict(FLEET_KNOBS)), fleet, transport=fake)
+
+ async def scenario():
+ task = asyncio.create_task(backend.run_skew())
+ try:
+ for _ in range(20):
+ await asyncio.sleep(0)
+ assert fake.calls == []
+ fleet.ready.set()
+ for _ in range(20):
+ await asyncio.sleep(0)
+ assert [c.path for c in fake.calls][:2] == ["model/info", "health"]
+ finally:
+ task.cancel()
+
+ asyncio.run(scenario())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org