You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/06 22:15:02 UTC
(tooling-llmao) branch main updated: fleet: an unknown box is approved with the models it should load
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
The following commit(s) were added to refs/heads/main by this push:
new ef02489 fleet: an unknown box is approved with the models it should load
ef02489 is described below
commit ef02489de464f33fd4cf4b4560f45ee9d3460878
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Tue Oct 6 17:14:46 2026 -0500
fleet: an unknown box is approved with the models it should load
A box that presents the fleet key but is not in fleet.hosts gets a
404 and a row in the attention strip. Approving it used to be a
label and nothing else: the label says the box is wanted, but the
next config fetch still 404s, because config.yaml has no row saying
which models to load.
Approving now answers that. The strip row has an Approve button that
opens one dialog listing every self-hosted model, each with a
checkbox. Approve stores the checked names for that IP and sets the
admin label to approved in the same call, so the label form is not a
second step. The strip row stays, and its detail line names the
models, until the IP is added to fleet.hosts.
THE NEXT FETCH IS SERVED
GET /vllm/config for an approved unknown host returns one server per
checked model, built the same way as a fleet.hosts row: listen ports
8001 upward, the server named after the model, no public port. An
unapproved host is still noted and 404s. A host already in
fleet.hosts keeps reading config.yaml; the approval is dropped the
moment the IP appears there, and when its unknown-fetch record is
evicted past the cap of 50.
NOTHING IS WRITTEN DOWN
approved_models is process memory, same as the unknown-fetch record
and the admin labels. config.yaml is not modified, and the box is
not added to Fleet.servers or to LiteLLM. An empty selection, an
unknown model, or a model that is not self-hosted is rejected and
leaves both stores untouched.
Co-authored-by: Grok <gr...@x.ai>
---
api.py | 2 +-
llmao/fleet.py | 57 +++++++++++++++++++++++++++++++++++++++---
pages.py | 62 ++++++++++++++++++++++++++++++++++++++++++++--
templates/models.ezt | 43 ++++++++++++++++++++++++++++++++
tests/test_fleet.py | 46 ++++++++++++++++++++++++++++++++++
tests/test_pages_models.py | 13 +++++++++-
6 files changed, 215 insertions(+), 8 deletions(-)
diff --git a/api.py b/api.py
index 5d6b265..280b51b 100644
--- a/api.py
+++ b/api.py
@@ -131,7 +131,7 @@ async def vllm_config():
claimed_host=request.headers.get(HOST_HEADER),
)
try:
- payload = config_for_host(host, cfg=APP.cfg)
+ payload = config_for_host(host, cfg=APP.cfg, approved=APP.fleet.approved_models.get(host))
except UnknownHostError:
APP.fleet.note_unknown_config_fetch(host)
raise HttpError(404, f"unknown fleet host: {host}") from None
diff --git a/llmao/fleet.py b/llmao/fleet.py
index 865b852..2263397 100644
--- a/llmao/fleet.py
+++ b/llmao/fleet.py
@@ -40,6 +40,7 @@ from easydict import EasyDict as edict # noqa: N813
from llmao.model_status import (
ADMIN_LABELS,
+ APPROVED,
AWAITING,
HEALTHY,
STALLED,
@@ -225,15 +226,26 @@ def config_for_host(
*,
models: list | None = None,
cfg: Any = None,
+ approved: list[str] | None = None,
) -> dict[str, Any]:
- """JSON for one host IP. Requires validate_fleet() already ran on cfg."""
+ """JSON for one host IP. Requires validate_fleet() already ran on cfg.
+
+ A host absent from fleet.hosts is served from ``approved``: one server
+ per model, listen ports 8001 upward, named after the model. That list
+ is the in-memory approval; config.yaml is not consulted and not written.
+ """
host = normalize_peer_ip(host)
- if not host or host not in cfg.fleet.hosts:
+ known = bool(host) and host in cfg.fleet.hosts
+ if not host or (not known and not approved):
raise UnknownHostError(host)
models = models if models is not None else load_models(cfg=cfg)
by_name = {model.model_name: model for model in models}
+ if known:
+ rows = list(cfg.fleet.hosts[host])
+ else:
+ rows = [[name, 8001 + i] for i, name in enumerate(approved)]
servers = []
- for i, raw in enumerate(cfg.fleet.hosts[host]):
+ for i, raw in enumerate(rows):
# The box is told its listen port; the public port is ours, not its
# business -- it binds inside the container.
model_name, port, name, _ = parse_host_row(raw, host, i)
@@ -556,6 +568,9 @@ class Fleet:
# computed lifecycle -- the page reads them for display. See admin_label.
self.admin_labels: dict[str, str] = {}
self.admin_label_at: dict[str, float] = {}
+ # Models an unknown box was told to load. Process memory only, same
+ # lifetime as the unknown-fetch record. config.yaml is not written.
+ self.approved_models: dict[str, list[str]] = {}
# LiteLLM routes that match no deployment. Rebuilt each config skew pass.
# Identity is host, port, and model name. api_base is not a primary key:
# one host serves several ports, and one host:port can report more than one model.
@@ -647,7 +662,36 @@ class Fleet:
rec = self.unknown_config_fetches.get(host)
if rec is None:
return ""
- return f"seen {_ago(rec['last_seen'])} · {int(rec['count'])} requests"
+ line = f"seen {_ago(rec['last_seen'])} · {int(rec['count'])} requests"
+ names = self.approved_models.get(normalize_peer_ip(host))
+ if names:
+ line += " · " + ", ".join(names)
+ return line
+
+ def approve_models(self, host: str, names: list[str]) -> None:
+ """Remember which models an unknown box should load, and mark it approved.
+
+ Replaces any previous list. Raises, changing nothing, when the host is
+ already in fleet.hosts, the selection is empty, or a name is unknown or
+ not self-hosted. Lost on restart.
+ """
+ host = normalize_peer_ip(host)
+ members = {normalize_peer_ip(str(ip)) for ip in self.cfg.fleet.hosts}
+ if host in members:
+ raise ValueError(f"{host} is already in fleet.hosts")
+ chosen = [str(n).strip() for n in names if str(n).strip()]
+ if not chosen:
+ raise ValueError("Select at least one model")
+ by_name = self.models or {m.model_name: m for m in load_models(cfg=self.cfg)}
+ for name in chosen:
+ model = by_name.get(name)
+ if model is None:
+ raise ValueError(f"Unknown model: {name}")
+ info = model.model_info
+ if not (info.get("self_hosted") if hasattr(info, "get") else False):
+ raise ValueError(f"{name} is not self-hosted")
+ self.approved_models[host] = chosen
+ self.set_admin_label(host, APPROVED)
def note_unknown_config_fetch(self, host: str, *, now: float | None = None) -> None:
"""Remember a fleet-key config fetch for an IP that is not in fleet.hosts.
@@ -661,6 +705,10 @@ class Fleet:
for known in list(self.unknown_config_fetches):
if known in members:
del self.unknown_config_fetches[known]
+ self.approved_models.pop(known, None)
+ for known in list(self.approved_models):
+ if known in members:
+ self.approved_models.pop(known, None)
if host in members:
return
rec = self.unknown_config_fetches.get(host)
@@ -677,6 +725,7 @@ class Fleet:
)
for ip in oldest[:extra]:
del self.unknown_config_fetches[ip]
+ self.approved_models.pop(ip, None)
def apply_port_map(self, mapping: Any) -> None:
"""Set public_port from Vast IP → listen → HostPort. Leave unset if missing.
diff --git a/pages.py b/pages.py
index 686ab13..c60e6be 100644
--- a/pages.py
+++ b/pages.py
@@ -50,7 +50,7 @@ from llmao.model_status import (
UNHEALTHY,
deployment_status,
)
-from llmao.models import model_available_for, model_in_service, ux_models
+from llmao.models import load_models, model_available_for, model_in_service, ux_models
from llmao.seam import AuthzError
# Replica summary order: the calm states first, the ones needing attention last.
@@ -322,6 +322,8 @@ def attention_items(fleet) -> list:
kind="unknown",
what=f"Fleet key presented from {host} (not in fleet.hosts)",
detail=fleet.unknown_fetch_detail(host),
+ approve=ezt.boolean(True),
+ approved=",".join(fleet.approved_models.get(host, [])),
**_label_fields(fleet, host),
)
)
@@ -334,12 +336,23 @@ def attention_items(fleet) -> list:
kind=view.lifecycle,
what=f"{dep.name} is {view.lifecycle.replace('_', ' ')}",
detail=dep.model_name,
+ approve=ezt.boolean(False),
+ approved="",
**_label_fields(fleet, ""),
)
)
for badge in dep.skew:
phrase = SKEW_PHRASE.get(badge, badge)
- items.append(edict(kind="skew", what=phrase, detail=dep.name, **_label_fields(fleet, "")))
+ items.append(
+ edict(
+ kind="skew",
+ what=phrase,
+ detail=dep.name,
+ approve=ezt.boolean(False),
+ approved="",
+ **_label_fields(fleet, ""),
+ )
+ )
for extra in fleet.extra_litellm:
for badge in extra.skew:
items.append(
@@ -347,6 +360,8 @@ def attention_items(fleet) -> list:
kind="skew",
what=SKEW_PHRASE.get(badge, badge),
detail=f"{extra.host}:{extra.port} {extra.model_name}",
+ approve=ezt.boolean(False),
+ approved="",
**_label_fields(fleet, ""),
)
)
@@ -408,9 +423,23 @@ async def models_page(result):
catalog = ux_models(cfg=APP.cfg, reveal_supply=admin)
result.models = model_catalog_rows(APP.fleet, catalog, admin=admin)
result.attention = attention_items(APP.fleet) if admin else []
+ result.approve_choices = _approve_choices(APP.cfg) if admin else []
return result
+def _approve_choices(cfg) -> list:
+ """Self-hosted models an unknown box can be told to load."""
+ choices = []
+ for model in load_models(cfg=cfg):
+ info = model.model_info
+ if not info.self_hosted:
+ continue
+ choices.append(
+ edict(model_name=str(model.model_name), display_name=str(info.get("display_name") or model.model_name))
+ )
+ return choices
+
+
def deployment_rows(fleet, *, admin: bool) -> list:
"""One table row per intended deployment, admin fields gated on ``admin``."""
rows = []
@@ -599,6 +628,35 @@ def apply_admin_label(fleet, host: str, label: str) -> tuple[bool, str]:
return True, f"Marked {host} as {label.replace('_', ' ')}."
+def apply_box_approval(fleet, host: str, names: list[str]) -> tuple[bool, str]:
+ """Remember the models for an unknown box. The testable core of /do-approve-box."""
+ host = (host or "").strip()
+ if not host:
+ return False, "No host to approve."
+ try:
+ fleet.approve_models(host, names)
+ except ValueError as e:
+ return False, str(e)
+ return True, f"Approved {host} to load {', '.join(fleet.approved_models[host])}."
+
+
+@APP.post("/do-approve-box")
+@asfquart.auth.require
+async def do_approve_box():
+ """Site admin picks the models an unknown box loads on its next config fetch."""
+ ident = await current_identity(APP.cfg)
+ if not ident.is_site_admin:
+ await flash_danger("Only site admins can approve a fleet box.")
+ return _see_other("/models")
+ form = await quart.request.form
+ ok, message = apply_box_approval(APP.fleet, form.get("host"), form.getlist("model"))
+ if ok:
+ await flash_success(message)
+ else:
+ await flash_danger(message)
+ return _see_other("/models")
+
+
@APP.post("/do-set-admin-label")
@asfquart.auth.require
async def do_set_admin_label():
diff --git a/templates/models.ezt b/templates/models.ezt
index 21e1bb4..95c4685 100644
--- a/templates/models.ezt
+++ b/templates/models.ezt
@@ -33,6 +33,11 @@
[end]
<button type="submit" class="btn btn-sm btn-outline-secondary ms-1">Set</button>
</form>
+ [if-any attention.approve]
+ <button type="button" class="btn btn-sm btn-outline-primary ms-1"
+ data-bs-toggle="modal" data-bs-target="#approveBoxModal"
+ data-host="[attention.host]" data-models="[attention.approved]">Approve</button>
+ [end]
[end]
</li>
[end]
@@ -317,4 +322,42 @@
</div>
</div>
</div>
+[if-any is_site_admin]
+<div class="modal fade" id="approveBoxModal" tabindex="-1" aria-labelledby="approveBoxModalLabel" aria-hidden="true">
+ <div class="modal-dialog">
+ <form class="modal-content" method="post" action="/do-approve-box">
+ <div class="modal-header">
+ <h2 class="modal-title fs-5" id="approveBoxModalLabel">Approve box</h2>
+ <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+ </div>
+ <div class="modal-body">
+ <p class="small text-muted">Choose the models this box loads the next time it asks for its config.</p>
+ <input type="hidden" name="host" id="approve-host" value="">
+ [for approve_choices]
+ <div class="form-check">
+ <input class="form-check-input" type="checkbox" name="model" value="[approve_choices.model_name]" id="approve-[approve_choices.model_name]">
+ <label class="form-check-label" for="approve-[approve_choices.model_name]">[approve_choices.display_name]</label>
+ </div>
+ [end]
+ </div>
+ <div class="modal-footer">
+ <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+ <button type="submit" class="btn btn-primary">Approve</button>
+ </div>
+ </form>
+ </div>
+</div>
+<script>
+document.getElementById("approveBoxModal").addEventListener("show.bs.modal", function (ev) {
+ var btn = ev.relatedTarget;
+ var host = btn.getAttribute("data-host") || "";
+ var picked = (btn.getAttribute("data-models") || "").split(",");
+ document.getElementById("approve-host").value = host;
+ document.getElementById("approveBoxModalLabel").textContent = "Approve " + host;
+ document.querySelectorAll("#approveBoxModal input[name=model]").forEach(function (cb) {
+ cb.checked = picked.indexOf(cb.value) !== -1;
+ });
+});
+</script>
+[end]
[include "footer.ezt"]
diff --git a/tests/test_fleet.py b/tests/test_fleet.py
index fe3641d..a7e4d8f 100644
--- a/tests/test_fleet.py
+++ b/tests/test_fleet.py
@@ -84,6 +84,52 @@ def test_unknown_host():
config_for_host("10.0.0.9", models=models, cfg=cfg)
+def _approvable(cfg, models):
+ fleet = Fleet(cfg=cfg, servers=[], models={m.model_name: m for m in models})
+ return fleet
+
+
+def test_approved_unknown_host_is_served():
+ cfg = _cfg({"127.0.0.1": [["gemma4-26b", 8001]]})
+ models = load_models(EXAMPLE)
+ fleet = _approvable(cfg, models)
+ fleet.approve_models("198.51.100.8", ["qwen3-8b", "gemma4-26b"])
+ assert fleet.admin_label("198.51.100.8").label == "approved"
+ payload = config_for_host(
+ "198.51.100.8",
+ models=models,
+ cfg=cfg,
+ approved=fleet.approved_models["198.51.100.8"],
+ )
+ assert [s["name"] for s in payload["servers"]] == ["qwen3-8b", "gemma4-26b"]
+ assert [s["port"] for s in payload["servers"]] == [8001, 8002]
+ assert payload["servers"][0]["model"] == "Qwen/Qwen3-8B-FP8"
+
+
+def test_approve_rejects_bad_selections():
+ cfg = _cfg({"127.0.0.1": [["gemma4-26b", 8001]]})
+ models = load_models(EXAMPLE)
+ fleet = _approvable(cfg, models)
+ fleet.models["claude"] = edict(model_name="claude", model_info=edict(self_hosted=False))
+ for names in ([], ["nope"], ["claude"]):
+ with pytest.raises(ValueError):
+ fleet.approve_models("198.51.100.8", names)
+ assert fleet.approved_models == {}
+ assert fleet.admin_labels == {}
+ with pytest.raises(ValueError, match="already in fleet"):
+ fleet.approve_models("127.0.0.1", ["gemma4-26b"])
+
+
+def test_approval_drops_when_the_host_joins_config():
+ cfg = _cfg({"127.0.0.1": [["gemma4-26b", 8001]]})
+ models = load_models(EXAMPLE)
+ fleet = _approvable(cfg, models)
+ fleet.approve_models("198.51.100.8", ["qwen3-8b"])
+ cfg.fleet.hosts["198.51.100.8"] = [["qwen3-8b", 8001]]
+ fleet.note_unknown_config_fetch("203.0.113.9", now=40)
+ assert "198.51.100.8" not in fleet.approved_models
+
+
def test_optional_name_two_copies():
cfg = _cfg(
{
diff --git a/tests/test_pages_models.py b/tests/test_pages_models.py
index 2c15dee..6e8fda9 100644
--- a/tests/test_pages_models.py
+++ b/tests/test_pages_models.py
@@ -327,6 +327,16 @@ def test_attention_strip_renders_the_label_form_for_unknown_boxes():
assert 'name="host" value="192.0.2.7"' in out
assert 'value="approved" selected' in out
assert "/do-set-admin-label" in out
+ assert 'data-bs-target="#approveBoxModal"' in out
+ out = _render(
+ rows,
+ attention=pages.attention_items(fleet),
+ admin=True,
+ approve_choices=[edict(model_name="gemma", display_name="Gemma")],
+ )
+ assert 'action="/do-approve-box"' in out
+ assert 'value="gemma"' in out
+ assert ">Gemma<" in out
def test_remove_refusal_gate():
@@ -375,7 +385,7 @@ def test_drill_down_renders_add_and_remove_actions():
assert "Remove only when vLLM is unhealthy or stalled" in out
-def _render(rows, attention=None, *, admin=False) -> str:
+def _render(rows, attention=None, *, admin=False, approve_choices=None) -> str:
data = edict(
title="Models",
is_site_admin=ezt.boolean(admin),
@@ -391,6 +401,7 @@ def _render(rows, attention=None, *, admin=False) -> str:
commit="deadbeef",
models=rows,
attention=attention or [],
+ approve_choices=approve_choices or [],
)
t = ezt.Template(str(THIS_DIR / "templates" / "models.ezt"))
buf = io.StringIO()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org