You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/08/05 17:05:20 UTC
(tooling-llmao) 09/17: Two-model self-host catalog: FP8 checkpoints, drop coder
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 617f6e7b478221101d94482b039b859aadec5972
Author: Andrew Musselman <an...@gmail.com>
AuthorDate: Wed Jul 8 09:34:40 2026 -0700
Two-model self-host catalog: FP8 checkpoints, drop coder
- catalog.py: point Gemma at RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic and the
8B at Qwen/Qwen3-8B-FP8 (pre-quantized). Document the FP8/eager/vLLM-0.19.1
requirements and the 8B thinking-mode note in each entry.
- catalog.py: remove the Qwen 3.6-27B coder from the resident set; leave a
comment recording it as a rotation candidate (Qwen/Qwen3.6-27B-FP8,
vLLM >=0.19.0) since one L40S cannot co-host three models this size.
- litellm/config.yaml: regenerate for the two remaining models.
---
infra/docker/docker-compose.yml | 6 +++---
litellm/config.yaml | 5 -----
llmao/catalog.py | 37 ++++++++++++++++---------------------
3 files changed, 19 insertions(+), 29 deletions(-)
diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml
index 699d08c..09b7a50 100644
--- a/infra/docker/docker-compose.yml
+++ b/infra/docker/docker-compose.yml
@@ -18,7 +18,7 @@
# capped per model so they co-reside. Tune the caps against real VRAM use.
x-vllm-common: &vllm-common
- image: vllm/vllm-openai:v0.11.0 # >=0.11.0 required for Qwen3.x / Gemma-4 architectures (older images fail with 'architecture not recognized')
+ image: vllm/vllm-openai:v0.19.1 # >=0.11.0 required for Qwen3.x / Gemma-4 architectures (older images fail with 'architecture not recognized')
restart: unless-stopped
networks: [llmao-net]
volumes:
@@ -98,7 +98,7 @@ services:
vllm-gemma:
<<: *vllm-common
command: >
- --model google/gemma-4-26b-a4b
+ --model RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic
--served-model-name gemma4-26b
--port 8001
--gpu-memory-utilization 0.38
@@ -120,7 +120,7 @@ services:
vllm-qwen8b:
<<: *vllm-common
command: >
- --model Qwen/Qwen3-8B
+ --model Qwen/Qwen3-8B-FP8
--served-model-name qwen3-8b
--port 8003
--gpu-memory-utilization 0.18
diff --git a/litellm/config.yaml b/litellm/config.yaml
index a7fda89..c07baf1 100644
--- a/litellm/config.yaml
+++ b/litellm/config.yaml
@@ -7,11 +7,6 @@ model_list:
model: openai/gemma4-26b
api_base: os.environ/LLMAO_SELFHOST_GEMMA_URL
api_key: os.environ/LLMAO_SELFHOST_API_KEY
- - model_name: selfhost/qwen3.6-27b
- litellm_params:
- model: openai/qwen3.6-27b
- api_base: os.environ/LLMAO_SELFHOST_QWEN_CODER_URL
- api_key: os.environ/LLMAO_SELFHOST_API_KEY
- model_name: selfhost/qwen3-8b
litellm_params:
model: openai/qwen3-8b
diff --git a/llmao/catalog.py b/llmao/catalog.py
index 620d11a..0b99e43 100644
--- a/llmao/catalog.py
+++ b/llmao/catalog.py
@@ -64,7 +64,7 @@ CATALOG: List[CatalogModel] = [
context_window=131072,
license="Apache-2.0",
openness="open-weight",
- weights_distribution="google/gemma-4-26b-a4b (HF) \u00b7 vLLM",
+ weights_distribution="RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic (HF) \u00b7 vLLM",
training_data_provenance="undisclosed (Google DeepMind)",
provenance_record="absent",
self_hosted=True,
@@ -72,25 +72,18 @@ CATALOG: List[CatalogModel] = [
served_name="gemma4-26b",
api_base_env="LLMAO_SELFHOST_GEMMA_URL",
notes="MoE, ~4B active. General/multimodal/agentic default. "
- "tau2-bench 85.5, LiveCodeBench v6 77.1 (vendor).",
- ),
- CatalogModel(
- id="self-host/qwen3.6-27b",
- display_name="Qwen 3.6 27B (self-hosted, coding)",
- provider="self-host",
- backend="selfhost/qwen3.6-27b",
- context_window=131072,
- license="Apache-2.0",
- openness="open-weight",
- weights_distribution="Qwen/Qwen3.6-27B (HF) \u00b7 vLLM",
- training_data_provenance="undisclosed (Alibaba)",
- provenance_record="absent",
- self_hosted=True,
- served_name="qwen3.6-27b",
- api_base_env="LLMAO_SELFHOST_QWEN_CODER_URL",
- notes="Best open dense coder in range. SWE-bench Verified 77.2, "
- "ties Sonnet 4.6 on agentic index (vendor).",
+ "Pre-quantized FP8 checkpoint (~28GB VRAM); requires vLLM "
+ ">=0.19.1 and --enforce-eager. tau2-bench 85.5, LiveCodeBench "
+ "v6 77.1 (vendor).",
),
+ # --- ROTATION CANDIDATE (not currently resident) ---------------------
+ # Qwen 3.6 27B coder was dropped from the resident set: one L40S (44 GiB
+ # usable) cannot co-host Gemma-4-26B (~28GB) + a 27B coder + the 8B at
+ # once, even at FP8. To offer it, either (a) run it via vLLM sleep-mode
+ # rotation (few-second wake cost), or (b) provision a second GPU.
+ # Confirmed available as a pre-quantized checkpoint that loads on our
+ # engine: Qwen/Qwen3.6-27B-FP8 (HF), vLLM >=0.19.0. Re-add a CatalogModel
+ # entry here when a rotation or larger-GPU story is in place.
CatalogModel(
id="self-host/qwen3-8b",
display_name="Qwen3 8B (self-hosted, fast)",
@@ -99,13 +92,15 @@ CATALOG: List[CatalogModel] = [
context_window=131072,
license="Apache-2.0",
openness="open-weight",
- weights_distribution="Qwen/Qwen3-8B (HF) \u00b7 vLLM",
+ weights_distribution="Qwen/Qwen3-8B-FP8 (HF) \u00b7 vLLM",
training_data_provenance="undisclosed (Alibaba)",
provenance_record="absent",
self_hosted=True,
served_name="qwen3-8b",
api_base_env="LLMAO_SELFHOST_QWEN8B_URL",
- notes="Fast lightweight tier for routine/cheap calls.",
+ notes="Fast lightweight tier for routine/cheap calls. Pre-quantized "
+ "FP8 (~8GB VRAM); loads on vLLM >=0.19.1 without --enforce-eager. "
+ "Reasoning model: set enable_thinking=false for fast/cheap use.",
),
]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org