You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/09/09 03:59:07 UTC
(tooling-llmao) branch main updated: Split vLLM listen vs public ports and rename fleet lifecycle states.
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
The following commit(s) were added to refs/heads/main by this push:
new 97f3ae8 Split vLLM listen vs public ports and rename fleet lifecycle states.
97f3ae8 is described below
commit 97f3ae853c3e087c34d82b1d7b6f18e6cd824359
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Tue Sep 8 22:59:02 2026 -0500
Split vLLM listen vs public ports and rename fleet lifecycle states.
Vast maps container ports to different public HostPorts, so one number cannot
drive both the box JSON (vLLM --port / install_set.py) and health/LiteLLM
api_base. Server now keeps listen_port for GET /vllm/config and public_port
for probes. Without fleet.vast, public equals listen (local). Unmapped
servers are not probed and stay pending.
States are pending / starting / serving / down (replacing unknown / healthy).
The health runner is fleet-lifecycle. /models still rolls up to Up, Starting,
Down, Mixed. /fleet shows listen and public columns.
No Vast API and no LiteLLM /model/new yet.
Co-authored-by: Grok <gr...@x.ai>
---
config.yaml.example | 2 ++
llmao/fleet.py | 59 ++++++++++++++++++++++---------------
llmao/litellm_client.py | 4 +--
main.py | 2 +-
pages.py | 8 ++++--
templates/fleet.ezt | 12 ++++----
tests/test_fleet_health.py | 72 +++++++++++++++++++++++++++++++++++++++-------
7 files changed, 115 insertions(+), 44 deletions(-)
diff --git a/config.yaml.example b/config.yaml.example
index 87844c7..02211f6 100644
--- a/config.yaml.example
+++ b/config.yaml.example
@@ -57,6 +57,8 @@ fleet:
skew_interval_s: 180
# Seconds between LiteLLM GET /health (real completions; tokens) vs fleet probe state.
litellm_health_interval_s: 14400
+ # Host row port is the container listen port (vLLM --port / box JSON).
+ # Without fleet.vast, the public port equals the listen port (local).
hosts:
127.0.0.1:
- [gemma4-26b, 8001]
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 3e5e07c..08a3cf4 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -156,9 +156,9 @@ def config_for_host(
class Server:
"""One vLLM process from fleet.hosts (live health + box JSON)."""
- UNKNOWN = "unknown"
+ PENDING = "pending"
STARTING = "starting"
- HEALTHY = "healthy"
+ SERVING = "serving"
DOWN = "down"
def __init__(
@@ -167,23 +167,25 @@ class Server:
model_name: str,
name: str,
host: str,
- port: int,
+ listen_port: int,
hf_model: str,
api_key: str,
args: list[str],
gpu_memory_utilization: float | None = None,
max_model_len: int | None = None,
+ public_port: int | None = None,
):
self.model_name = model_name
self.name = name
self.host = host
- self.port = int(port)
+ self.listen_port = int(listen_port)
+ self.public_port = int(public_port) if public_port is not None else None
self.hf_model = hf_model
self.api_key = api_key
self.args = args
self.gpu_memory_utilization = gpu_memory_utilization
self.max_model_len = max_model_len
- self.state = self.UNKNOWN
+ self.state = self.PENDING
self.seen_at = time.time()
self.last_ok = None
self.last_error = None
@@ -202,7 +204,7 @@ class Server:
model_name=model_name,
name=name,
host=host,
- port=port,
+ listen_port=port,
hf_model=str(vllm.model),
# Per-instance credential, not a catalog property -- generated when
# a host is added, same reasoning as api_base. Empty until the
@@ -214,12 +216,16 @@ class Server:
)
@property
- def health_url(self) -> str:
- return f"http://{self.host}:{self.port}/health"
+ def health_url(self) -> str | None:
+ if self.public_port is None:
+ return None
+ return f"http://{self.host}:{self.public_port}/health"
@property
- def api_base(self) -> str:
- return f"http://{self.host}:{self.port}"
+ def api_base(self) -> str | None:
+ if self.public_port is None:
+ return None
+ return f"http://{self.host}:{self.public_port}"
def box_json(self) -> dict[str, Any]:
"""Wire payload for GET /vllm/config (servers[].model is the HF weights id)."""
@@ -227,7 +233,7 @@ class Server:
"name": self.name,
"model": self.hf_model,
"host": self.host,
- "port": self.port,
+ "port": self.listen_port,
"api_key": self.api_key,
"args": list(self.args),
}
@@ -247,16 +253,16 @@ class Server:
err: str | None = None,
) -> None:
if ok:
- if self.state != self.HEALTHY:
- _LOGGER.info(f"fleet server {self.name}@{self.api_base} healthy")
- self.state = self.HEALTHY
+ if self.state != self.SERVING:
+ _LOGGER.info(f"fleet server {self.name}@{self.api_base} serving")
+ self.state = self.SERVING
self.last_ok = now
self.last_error = None
self.fails = 0
return
self.fails += 1
self.last_error = err or "unhealthy"
- if self.state == self.HEALTHY:
+ if self.state == self.SERVING:
if self.fails >= fail_threshold:
self.state = self.DOWN
_LOGGER.warning(
@@ -274,7 +280,7 @@ class Server:
class Fleet:
- """Live fleet: servers from config, health from probes, config-fetch stamps."""
+ """Live fleet: servers from config, lifecycle probes, config-fetch stamps."""
BADGE_UP = "up"
BADGE_STARTING = "starting"
@@ -290,12 +296,16 @@ class Fleet:
def from_cfg(cls, cfg: Any, models: list | None = None) -> Fleet:
models = models if models is not None else load_model_list(cfg=cfg)
by_name = {model.model_name: model for model in models}
+ local = "vast" not in cfg.fleet
servers = []
for host, rows in cfg.fleet.hosts.items():
host = str(host).strip()
for i, raw in enumerate(rows):
model_name, port, name = parse_host_row(raw, host, i)
- servers.append(Server.from_row(host, model_name, port, name, by_name[model_name]))
+ srv = Server.from_row(host, model_name, port, name, by_name[model_name])
+ if local:
+ srv.public_port = srv.listen_port
+ servers.append(srv)
return cls(cfg, servers)
def note_config_fetch(self, host: str, *, now: float | None = None) -> None:
@@ -307,13 +317,13 @@ class Fleet:
if not states:
return ""
uniq = set(states)
- if uniq == {Server.HEALTHY}:
+ if uniq == {Server.SERVING}:
return self.BADGE_UP
- if uniq <= {Server.STARTING, Server.UNKNOWN}:
+ if uniq <= {Server.STARTING, Server.PENDING}:
return self.BADGE_STARTING
if uniq == {Server.DOWN}:
return self.BADGE_DOWN
- if Server.HEALTHY in uniq and uniq <= {Server.HEALTHY, Server.STARTING, Server.UNKNOWN}:
+ if Server.SERVING in uniq and uniq <= {Server.SERVING, Server.STARTING, Server.PENDING}:
return self.BADGE_UP
return self.BADGE_MIXED
@@ -332,7 +342,10 @@ class Fleet:
client = httpx.AsyncClient(timeout=httpx.Timeout(timeout))
try:
for srv in self.servers:
- ok, err = await _get_health(client, srv.health_url)
+ url = srv.health_url
+ if url is None:
+ continue
+ ok, err = await _get_health(client, url)
srv.record_probe(
ok, now=stamp, grace_s=grace, fail_threshold=threshold, err=err
)
@@ -340,13 +353,13 @@ class Fleet:
if own:
await client.aclose()
- async def run_health(self) -> None:
+ async def run_lifecycle(self) -> None:
interval = float(self.cfg.fleet.health_interval_s)
while True:
try:
await self.probe_all()
except Exception:
- _LOGGER.exception("fleet health probe failed")
+ _LOGGER.exception("fleet lifecycle probe failed")
await asyncio.sleep(interval)
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index cfca59c..79b0856 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -496,7 +496,7 @@ class LiteLLMBackend:
resp = await self._request("GET", "model/info")
self._raise_http(resp)
bases = _api_bases_from_model_info(resp.json())
- fleet_bases = {s.api_base for s in self.fleet.servers}
+ fleet_bases = {s.api_base for s in self.fleet.servers if s.api_base}
for srv in self.fleet.servers:
note = "missing from LiteLLM"
if srv.api_base in bases:
@@ -519,7 +519,7 @@ class LiteLLMBackend:
litellm_down = srv.api_base in unhealthy
note = "LiteLLM health disagrees"
disagrees = (
- (srv.state == Server.HEALTHY and litellm_down)
+ (srv.state == Server.SERVING and litellm_down)
or (srv.state == Server.DOWN and litellm_up)
)
if disagrees:
diff --git a/main.py b/main.py
index 6c8e6f6..6777977 100644
--- a/main.py
+++ b/main.py
@@ -65,7 +65,7 @@ def create_app():
backend = LiteLLMBackend(app.cfg, fleet)
app.fleet = fleet
- app.add_runner(fleet.run_health, name="fleet-health")
+ app.add_runner(fleet.run_lifecycle, name="fleet-lifecycle")
app.add_runner(backend.run_skew, name="litellm-skew")
seam = Seam(app.cfg, backend)
app.token_handler = make_token_handler(app.cfg)
diff --git a/pages.py b/pages.py
index ef380ba..d9d2791 100644
--- a/pages.py
+++ b/pages.py
@@ -226,14 +226,18 @@ async def fleet_page(result):
rows.append(edict({
"host": srv.host,
"name": srv.name,
- "host_port": f"{srv.host}:{srv.port}",
+ "listen": f"{srv.host}:{srv.listen_port}",
+ "public": (
+ f"{srv.host}:{srv.public_port}" if srv.public_port is not None else "—"
+ ),
"state": srv.state,
"last_ok": _ago(srv.last_ok, now),
"config_ago": _ago(fetched, now),
"skew": "; ".join(srv.skew) if srv.skew else "",
- "healthy": ezt.boolean(srv.state == Server.HEALTHY),
+ "serving": ezt.boolean(srv.state == Server.SERVING),
"starting": ezt.boolean(srv.state == Server.STARTING),
"down": ezt.boolean(srv.state == Server.DOWN),
+ "pending": ezt.boolean(srv.state == Server.PENDING),
}))
result.servers = rows
return result
diff --git a/templates/fleet.ezt b/templates/fleet.ezt
index b720378..563434b 100644
--- a/templates/fleet.ezt
+++ b/templates/fleet.ezt
@@ -18,6 +18,7 @@
<th>Host</th>
<th>Server</th>
<th>Listen</th>
+ <th>Public</th>
<th>State</th>
<th>Last ok</th>
<th>Config request</th>
@@ -29,9 +30,10 @@
<tr>
<td class="font-monospace small">[servers.host]</td>
<td>[servers.name]</td>
- <td class="font-monospace small">[servers.host_port]</td>
+ <td class="font-monospace small">[servers.listen]</td>
+ <td class="font-monospace small">[servers.public]</td>
<td>
- [if-any servers.healthy]
+ [if-any servers.serving]
<span class="badge text-bg-success">[servers.state]</span>
[end]
[if-any servers.starting]
@@ -40,11 +42,9 @@
[if-any servers.down]
<span class="badge text-bg-danger">[servers.state]</span>
[end]
- [if-any servers.healthy][else]
- [if-any servers.starting][else]
- [if-any servers.down][else]
+ [if-any servers.pending]
<span class="badge text-bg-light text-dark">[servers.state]</span>
- [end][end][end]
+ [end]
</td>
<td class="small">[servers.last_ok]</td>
<td class="small">[servers.config_ago]</td>
diff --git a/tests/test_fleet_health.py b/tests/test_fleet_health.py
index 7d076b9..a3bdd48 100644
--- a/tests/test_fleet_health.py
+++ b/tests/test_fleet_health.py
@@ -1,4 +1,8 @@
"""Fleet Server health state machine (no real vLLM)."""
+import asyncio
+
+from easydict import EasyDict as edict
+
from llmao.fleet import Fleet, Server
@@ -7,19 +11,20 @@ def _server(**kwargs):
model_name="gemma4-26b",
name="gemma4-26b",
host="10.0.0.1",
- port=8001,
+ listen_port=8001,
hf_model="google/gemma",
api_key="sk-x",
args=[],
+ public_port=8001,
)
defaults.update(kwargs)
return Server(**defaults)
-def test_probe_healthy():
+def test_probe_serving():
s = _server()
s.record_probe(True, now=100.0, grace_s=1800, fail_threshold=3)
- assert s.state == Server.HEALTHY
+ assert s.state == Server.SERVING
assert s.last_ok == 100.0
@@ -37,20 +42,20 @@ def test_down_after_grace():
assert s.state == Server.DOWN
-def test_healthy_needs_consecutive_fails():
+def test_serving_needs_consecutive_fails():
s = _server()
s.record_probe(True, now=1.0, grace_s=1800, fail_threshold=3)
s.record_probe(False, now=2.0, grace_s=1800, fail_threshold=3)
s.record_probe(False, now=3.0, grace_s=1800, fail_threshold=3)
- assert s.state == Server.HEALTHY
+ assert s.state == Server.SERVING
s.record_probe(False, now=4.0, grace_s=1800, fail_threshold=3)
assert s.state == Server.DOWN
def test_model_health_aggregate():
- a = _server(name="a", port=1)
- b = _server(name="b", port=2, model_name="qwen3-8b")
- a.state = Server.HEALTHY
+ a = _server(name="a", listen_port=1)
+ b = _server(name="b", listen_port=2, model_name="qwen3-8b")
+ a.state = Server.SERVING
b.state = Server.STARTING
fleet = Fleet(cfg=None, servers=[a, b])
assert fleet.model_health("gemma4-26b") == Fleet.BADGE_UP
@@ -64,9 +69,56 @@ def test_note_config_fetch():
assert fleet.config_fetch_at["primary"] == 42.0
-def test_box_json_method():
- s = _server(gpu_memory_utilization=0.5)
+def test_box_json_listen_not_public():
+ s = _server(gpu_memory_utilization=0.5, public_port=41234)
body = s.box_json()
assert body["model"] == "google/gemma"
assert body["port"] == 8001
assert body["gpu_memory_utilization"] == 0.5
+ assert s.api_base == "http://10.0.0.1:41234"
+
+
+def test_local_from_cfg_public_equals_listen():
+ from pathlib import Path
+ from llmao.models import load_model_list
+
+ example = Path(__file__).resolve().parent.parent / "model_list.yaml.example"
+ cfg = edict({
+ "fleet": {
+ "hosts": {"127.0.0.1": [["gemma4-26b", 8001]]},
+ "health_interval_s": 45,
+ "health_timeout_s": 3,
+ "health_grace_s": 1800,
+ "health_fail_threshold": 3,
+ "skew_interval_s": 180,
+ "litellm_health_interval_s": 14400,
+ },
+ "models_path": str(example),
+ })
+ fleet = Fleet.from_cfg(cfg, models=load_model_list(example))
+ assert fleet.servers[0].listen_port == 8001
+ assert fleet.servers[0].public_port == 8001
+ assert fleet.servers[0].state == Server.PENDING
+
+
+def test_probe_skips_without_public_port():
+ s = _server(public_port=None)
+ assert s.health_url is None
+ cfg = edict({
+ "fleet": edict({
+ "health_timeout_s": 1,
+ "health_grace_s": 1800,
+ "health_fail_threshold": 3,
+ })
+ })
+
+ class _Boom:
+ async def get(self, url):
+ raise AssertionError(f"must not probe {url}")
+
+ async def aclose(self):
+ return None
+
+ fleet = Fleet(cfg=cfg, servers=[s])
+ asyncio.run(fleet.probe_all(client=_Boom(), now=1.0))
+ assert s.state == Server.PENDING
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org