You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/08/05 17:05:28 UTC
(tooling-llmao) 17/17: Adding thinking and temperature elements in the portal
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 22566fc755a82852b2022b20ebbdb49954ea397d
Author: Andrew Musselman <an...@gmail.com>
AuthorDate: Mon Aug 3 18:53:26 2026 -0700
Adding thinking and temperature elements in the portal
---
llmao/app.py | 23 ++++++++++++++++++++-
llmao/catalog.py | 18 +++++++++++++++++
llmao/litellm_client.py | 13 +++++++++++-
llmao/portal.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++--
4 files changed, 103 insertions(+), 4 deletions(-)
diff --git a/llmao/app.py b/llmao/app.py
index 7ddf311..d8a513a 100644
--- a/llmao/app.py
+++ b/llmao/app.py
@@ -147,6 +147,20 @@ def create_app(settings: Optional[Settings] = None) -> Quart:
params = {k: v for k, v in body.items() if k in ("temperature", "top_p", "max_tokens", "stream")}
params.pop("stream", None) # Phase 1 returns non-streamed; streaming is a later toggle.
+ # Extended thinking. Sent as vLLM's chat-template kwarg rather than a
+ # sampling parameter, so it needs its own path -- the whitelist above
+ # is deliberately narrow and would drop it.
+ #
+ # Only forwarded when the model declares supports_thinking, and only
+ # when the caller was explicit: passing the kwarg at all overrides the
+ # model's own default, and those defaults differ (gemma4 off, qwen3
+ # on). Omitting it leaves each model's native behaviour intact.
+ think = body.get("thinking")
+ if think is not None:
+ model = catalog.get(model_id)
+ if model is not None and getattr(model, "supports_thinking", False):
+ params["chat_template_kwargs"] = {"enable_thinking": bool(think)}
+
try:
completion = seam.chat(ident, project, model_id, messages, params)
except AuthzError as e:
@@ -166,7 +180,14 @@ def create_app(settings: Optional[Settings] = None) -> Quart:
"llmao_project": project,
"choices": [{
"index": 0,
- "message": {"role": "assistant", "content": completion.content},
+ "message": {
+ "role": "assistant",
+ "content": completion.content,
+ # Present only when the model reasoned. Rendered collapsed
+ # in the portal -- diagnostic, not part of the answer.
+ **({"reasoning_content": completion.reasoning}
+ if getattr(completion, "reasoning", None) else {}),
+ },
"finish_reason": "stop",
}],
"usage": {
diff --git a/llmao/catalog.py b/llmao/catalog.py
index fb8d9ce..6ec46ca 100644
--- a/llmao/catalog.py
+++ b/llmao/catalog.py
@@ -43,6 +43,20 @@ class CatalogModel:
served_name: Optional[str] = None # vLLM --served-model-name
api_base_env: Optional[str] = None # env var holding this model's vLLM URL
+ # --- thinking / extended reasoning -----------------------------------
+ # supports_thinking: the served model runs a reasoning parser, so the
+ # enable_thinking chat-template kwarg is meaningful. False hides the
+ # toggle in the portal and the kwarg is never sent.
+ #
+ # thinks_by_default: what the model does with NO kwarg. Differs per model
+ # and is not guessable -- measured 2026-08-04 through the proxy:
+ # qwen3-8b with no kwarg -> 893 chars of reasoning (True)
+ # gemma4-26b with no kwarg -> 0 chars (False)
+ # Gemma reasons only when asked; Qwen reasons unless told not to. A
+ # single global default would be wrong for one of them.
+ supports_thinking: bool = False
+ thinks_by_default: bool = False
+
def public(self) -> Dict:
d = asdict(self)
# Serving details are operational, not part of the public catalog.
@@ -81,6 +95,8 @@ CATALOG: List[CatalogModel] = [
modality="text+vision",
served_name="gemma4-26b",
api_base_env="LLMAO_SELFHOST_GEMMA_URL",
+ supports_thinking=True,
+ thinks_by_default=False, # --reasoning-parser gemma4 leaves it off
notes="MoE, ~4B active. General/multimodal/agentic default. "
"Served off-host at BF16 on an 80GB card (~49GB weights, "
"131072 window, ~856k-token KV cache). Measured ~100 tok/s "
@@ -109,6 +125,8 @@ CATALOG: List[CatalogModel] = [
self_hosted=True,
served_name="qwen3-8b",
api_base_env="LLMAO_SELFHOST_QWEN8B_URL",
+ supports_thinking=True,
+ thinks_by_default=True, # reasons unless enable_thinking=false
notes="Fast lightweight tier for routine/cheap calls. Sole resident "
"on the local L40S at FP8 (~8GB weights). Reasoning model: "
"vLLM runs --reasoning-parser qwen3, and unlike Gemma the "
diff --git a/llmao/litellm_client.py b/llmao/litellm_client.py
index 3b4031f..14107b2 100644
--- a/llmao/litellm_client.py
+++ b/llmao/litellm_client.py
@@ -42,6 +42,11 @@ class Completion:
prompt_tokens: int
completion_tokens: int
cost_usd: float
+ # Populated when the model reasoned. vLLM's reasoning parsers split the
+ # thinking block out of `content` into a separate field, so without this
+ # the portal shows an answer with no sign of the tokens spent producing
+ # it -- and at a low max_tokens, an EMPTY answer with no explanation.
+ reasoning: Optional[str] = None
@dataclass
@@ -244,7 +249,12 @@ class ProxyBackend:
raise BudgetExceeded(resp.text)
resp.raise_for_status()
body = resp.json()
- choice = body["choices"][0]["message"]["content"]
+ message = body["choices"][0]["message"]
+ choice = message.get("content")
+ # vLLM emits `reasoning`; litellm normalises it to `reasoning_content`.
+ # Accept either -- which one arrives depends on the proxy version, and
+ # a silent None here looks identical to "the model didn't think".
+ reasoning = message.get("reasoning_content") or message.get("reasoning")
usage = body.get("usage", {})
cost = float(body.get("_hidden_params", {}).get("response_cost") or 0.0)
@@ -259,6 +269,7 @@ class ProxyBackend:
return Completion(
choice, model_backend,
usage.get("prompt_tokens", 0), usage.get("completion_tokens", 0), cost,
+ reasoning=reasoning,
)
def usage(self, project: Optional[str]) -> List[Dict]:
diff --git a/llmao/portal.py b/llmao/portal.py
index 0757203..d7876a2 100644
--- a/llmao/portal.py
+++ b/llmao/portal.py
@@ -122,6 +122,15 @@ def render_portal(settings: Settings, ident: Optional[Identity], models: List[Di
<label>Prompt
<textarea id="prompt" rows="6" placeholder="Ask something, or attach a file below."></textarea>
</label>
+ <div class="row2">
+ <label>Temperature <span class="hint" id="temp-val">0.6</span>
+ <input type="range" id="temperature" min="0" max="1" step="0.1" value="0.6">
+ </label>
+ <label class="think" id="think-wrap">
+ <input type="checkbox" id="thinking"> Extended thinking
+ <span class="hint" id="think-hint"></span>
+ </label>
+ </div>
<div class="row2">
<label class="file">Attach a file <span class="hint">text · md · code · ≤2 MB</span>
<input type="file" id="file" accept=".txt,.md,.json,.py,.java,.go,.yaml,.yml,.csv,.log">
@@ -223,6 +232,13 @@ textarea{resize:vertical;line-height:1.5;}
.bar>span{display:block;height:100%;background:var(--accent);}
.bar.warn>span{background:var(--warn);}
.bar.bad>span{background:var(--bad);}
+.think{display:flex;align-items:center;gap:8px;font-size:13px;}
+.think input{width:auto;margin:0;}
+input[type=range]{width:100%;accent-color:var(--accent, #5b9dd9);}
+.reasoning{margin-top:10px;border-top:1px solid var(--line);padding-top:8px;}
+.reasoning summary{cursor:pointer;color:var(--faint);font-size:12px;user-select:none;}
+.reasoning pre{white-space:pre-wrap;color:var(--dim);font-size:12px;
+ font-family:var(--mono);margin:8px 0 0;}
.activity{font-family:var(--mono);font-size:12px;color:var(--dim);}
.activity table{width:100%;border-collapse:collapse;margin-top:8px;}
.activity th{text-align:left;color:var(--faint);font-weight:500;border-bottom:1px solid var(--line);
@@ -244,6 +260,7 @@ function fillModels(){
function showMeta(){
const m = CTX.models.find(x => x.id === modelSel.value);
if(!m){ $('modelmeta').textContent=''; return; }
+ syncThinking(m);
const openCls = m.openness === 'open-weight' || m.openness === 'open-source' ? 'lic' : 'prop';
$('modelmeta').innerHTML =
`<b>license</b> <span class="${openCls}">${m.license}</span> · <b>openness</b> ${m.openness}<br>`+
@@ -297,6 +314,23 @@ async function readFile(file){
});
}
+// The thinking toggle reflects each model's OWN default, because they
+// differ: gemma4 does not reason unless asked, qwen3 reasons unless told
+// not to. Showing one fixed default would misrepresent one of them, and a
+// user who never touches the box would get behaviour they did not choose.
+function syncThinking(m){
+ const wrap=$('think-wrap'), box=$('thinking'), hint=$('think-hint');
+ if(!m.supports_thinking){
+ wrap.style.display='none';
+ box.checked=false;
+ return;
+ }
+ wrap.style.display='flex';
+ box.checked = !!m.thinks_by_default;
+ hint.textContent = m.thinks_by_default ? '(on by default for this model)'
+ : '(off by default for this model)';
+}
+
async function send(){
const btn=$('send'), out=$('out');
const project=projectSel.value, model=modelSel.value;
@@ -312,7 +346,13 @@ async function send(){
try{
const r=await fetch('/v1/chat/completions',{
method:'POST', headers:{'Content-Type':'application/json','X-LLMAO-Project':project},
- body:JSON.stringify({model, messages:[{role:'user',content:prompt}]}),
+ body:JSON.stringify({
+ model, messages:[{role:'user',content:prompt}],
+ temperature: parseFloat($('temperature').value),
+ // Always sent when the model supports it, so the checkbox is
+ // authoritative rather than only overriding in one direction.
+ ...($('think-wrap').style.display==='none' ? {} : {thinking: $('thinking').checked}),
+ }),
});
// Parse defensively: a timeout or crash can yield a non-JSON body.
let j=null;
@@ -328,7 +368,15 @@ async function send(){
else{
const u=j.usage;
out.className='out';
- out.innerHTML = escapeHtml(j.choices[0].message.content) +
+ const msg = j.choices[0].message;
+ // Collapsed by default -- the reasoning is diagnostic, not something
+ // anyone reads by choice. It is here so a surprising answer can be
+ // traced, not to be part of the normal reading flow.
+ const think = msg.reasoning_content
+ ? `<details class="reasoning"><summary>Thinking</summary>`
+ + `<pre>${escapeHtml(msg.reasoning_content)}</pre></details>`
+ : '';
+ out.innerHTML = escapeHtml(msg.content || '') + think +
`<div class="usage">${u.total_tokens} tokens · $${(u.cost_usd||0).toFixed(4)} · billed to ${j.llmao_project}</div>`;
}
}catch(e){ out.className='out err'; out.textContent=String(e); }
@@ -336,6 +384,7 @@ async function send(){
}
function escapeHtml(s){return s.replace(/[&<>]/g,c=>({'&':'&','<':'<','>':'>'}[c]));}
+$('temperature').addEventListener('input', e => { $('temp-val').textContent = e.target.value; });
modelSel.addEventListener('change', showMeta);
projectSel.addEventListener('change', refreshBudget);
$('send').addEventListener('click', send);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org